[ Sep 9 17:49:05 Enabled. ] [ Sep 9 17:49:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Sep 9 17:49:05 Method "start" exited with status 0. ] [ Sep 9 17:49:05 Rereading configuration. ] [ Sep 9 17:49:05 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:05.869424476Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:05.869569097Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:05.921045719Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:05.921094959Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.041203713Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.051126001Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.059197268Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.663253727Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.80199274Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.802052331Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.802069061Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.802169621Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.802190111Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.802224692Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805564171Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:06.805573041Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805584921Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805589961Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805627872Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805657792Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805775372Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805808993Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.805820573Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.816730526Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.816755137Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"d05e61ff-a727-4851-80ca-eab919525a00 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:06.934270075Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414842861Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414879371Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414885991Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414899851Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414908781Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414940831Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414945622Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414952022Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414956951Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.414964551Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.423417031Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.423442261Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:07.533365845Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.012947425Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.012986135Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013009595Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013015955Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013020565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013026295Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013031435Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013036875Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013061995Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013066826Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013070656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013075436Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013080966Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013096186Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013112676Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013116886Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013152436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013157156Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013203026Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013216176Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013244087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013248287Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013252187Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013256937Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013260137Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013263547Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013282467Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013286637Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013290237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013297127Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013301207Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013307267Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013332077Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013336437Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013341507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013344697Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013354037Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013382177Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013386537Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013389617Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013394177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013398288Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013413808Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013425978Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013456078Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013460648Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013464568Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013468738Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013472948Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013477158Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013480618Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013510708Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013515338Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013519278Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013522458Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013533048Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013536538Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013564938Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013569278Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013574078Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013582889Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013645339Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013656239Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013663869Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013674679Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013681509Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013685389Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013712859Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.013716959Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02414952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:730"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.024178971Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99993284s","next_cleanup":"Instant { tv_sec: 34069, tv_nsec: 79059468 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.024251421Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.024256871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.024313812Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.024343402Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027360709Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02742424Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027634051Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1053"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027645611Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027724872Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027767602Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027791372Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027795982Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027817912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027829492Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027836772Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027841052Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027844542Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027848022Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027874412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.027901102Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028205204Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 1068bc7e-e3ab-44f6-b097-299854b0081f,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 61e07fcc-45b4-4b25-ae0d-e78eee62eb60,\n rack_id: 1068bc7e-e3ab-44f6-b097-299854b0081f,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028488836Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028493696Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1187"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028530476Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028541266Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028728347Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028859048Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.028895778Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 61e07fcc-45b4-4b25-ae0d-e78eee62eb60, rack_id: 1068bc7e-e3ab-44f6-b097-299854b0081f, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.029004389Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:42816","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 61e07fcc-45b4-4b25-ae0d-e78eee62eb60, rack_id: 1068bc7e-e3ab-44f6-b097-299854b0081f, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02909578Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02910516Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02913002Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02913693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02914031Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02914339Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02916029Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02916831Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02918204Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02918637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02918942Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02919421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02922359Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.02922955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.031587064Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4163"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.031634104Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.06156854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.082340332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.098986569Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.107834961Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:388"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.107871661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.107881991Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:400"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.107933391Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 74.392977ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.11103894Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9587c68f-9ed8-4514-8ce1-e43a7c989fd2\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9587c68f-9ed8-4514-8ce1-e43a7c989fd2\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_d05e61ff-a727-4851-80ca-eab919525a00/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.121649082Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_d05e61ff-a727-4851-80ca-eab919525a00/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.147704284Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_d05e61ff-a727-4851-80ca-eab919525a00/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.147747435Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.152923825Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.152931645Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:127"} {"msg":"Ensuring dataset oxi_d05e61ff-a727-4851-80ca-eab919525a00/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.152943695Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:137"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.186897914Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 102.794585ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.187816159Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b7f7d51-caee-4183-8d91-71af36654c86\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b7f7d51-caee-4183-8d91-71af36654c86\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.258697455Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:172"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.291809749Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 175.285683ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.292316732Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e2b7901-dfd2-4a90-9e4a-c2e9a451d984\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e2b7901-dfd2-4a90-9e4a-c2e9a451d984\" }"} {"msg":"Mounting oxi_d05e61ff-a727-4851-80ca-eab919525a00/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.377732082Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:180"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.394653931Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.394730702Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.394745522Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.394762382Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:194"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.468172732Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 183.003904ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.468880296Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34f93ef3-66a1-422e-bd08-bf1e93412bdd\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34f93ef3-66a1-422e-bd08-bf1e93412bdd\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.652742134Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 268.840739ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.653408747Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6f0eaebb-6551-4f3b-abcb-f26409ad2ca7\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6f0eaebb-6551-4f3b-abcb-f26409ad2ca7\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.923819032Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 195.657973ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:08.924689997Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac54d7b9-1e16-4725-871b-68aa9abd35e0\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac54d7b9-1e16-4725-871b-68aa9abd35e0\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.114560529Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.120201862Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.120236162Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.120298013Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:42327","producer_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.121147208Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:09.121672091Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"203.032926ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.122490916Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.122497626Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.122582476Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:914","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:09.122587956Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:09.32630104Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"423.731904ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:09.751889674Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.079422325s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:10.832730375Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.844026654s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681122856Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2639"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681148686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2857"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681154356Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2509"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681161796Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2556"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681166136Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2938"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.681170286Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.683187608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2782"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.684948728Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2550"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.685104779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:11.68522851Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 61.026599ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:11.947675558Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 74.933565ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:12.128714808Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 149.408513ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:12.311248588Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 499.404536ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:12.49262355Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 690.923478ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:13.052117908Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:13.678652709Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.16935289s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 543.532017ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:13.803924683Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 975.535938ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:14.42985984Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 822.898411ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:15.590369949Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 766.861436ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:16.53947706Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.576959818Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577005888Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577017038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577041158Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.57728014Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577446601Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577500301Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:42816","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577516431Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577524181Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577571411Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.577582281Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863209735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863278475Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863291165Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863325776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863343966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.863354216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.868360525Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4187","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.868381655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4203"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:17.868396635Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.164442149Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60350"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.164675001Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.164763741Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"7bd72791-e2af-41c6-a06c-a4d1e99bee76","remote_addr":"[fd00:1122:3344:101::1]:60350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":212,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.195102669Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.19518171Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"bff3fc1f-8269-4508-b125-a761a8a9b98a","remote_addr":"[fd00:1122:3344:101::1]:39666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.195914754Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.195941244Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.196329026Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.196588948Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:1001"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 99445748-3f8a-47eb-b03d-540bb23b7924, pool_id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: eaf34cef-e592-4e7c-b9f3-1a240bd6bcf6, pool_id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: b3e70874-57ef-4fcb-9f44-a18b6a655d68, pool_id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 3cca5293-4433-4956-b75c-0e6967d69a04, pool_id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 032350b3-5ef6-49e0-82b7-c2f6c5c3d580, pool_id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 8d6a3fc4-42ae-4137-8a4c-0c429a53310f, pool_id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 31811878-08d9-4dd7-82d9-61bcc051e746, pool_id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6c038f91-337d-4974-956d-da4fd59787a8, pool_id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: e6df29c4-987d-4c4b-8971-6886a64153eb, pool_id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236349471Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236483712Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33471"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236758903Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:856"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 99445748-3f8a-47eb-b03d-540bb23b7924, pool_id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: eaf34cef-e592-4e7c-b9f3-1a240bd6bcf6, pool_id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: b3e70874-57ef-4fcb-9f44-a18b6a655d68, pool_id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 3cca5293-4433-4956-b75c-0e6967d69a04, pool_id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 032350b3-5ef6-49e0-82b7-c2f6c5c3d580, pool_id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 8d6a3fc4-42ae-4137-8a4c-0c429a53310f, pool_id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 31811878-08d9-4dd7-82d9-61bcc051e746, pool_id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6c038f91-337d-4974-956d-da4fd59787a8, pool_id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: e6df29c4-987d-4c4b-8971-6886a64153eb, pool_id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236799613Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236805764Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236815033Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236819373Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236828474Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236832514Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236837374Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.236841214Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.38474766Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:18.38478824Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:18.656591153Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:18.895364442Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.627662798s"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.598787773Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.617153581Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.625823412Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.625894502Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.625903832Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.625911412Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.625918302Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.626025483Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.626033243Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:19.659269848Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.668466212Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e68e093e-0654-4278-a79f-9c234ab4cc35/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.676008376Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_e68e093e-0654-4278-a79f-9c234ab4cc35","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.88663099Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.894849548Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.894887058Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.894894448Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.894903198Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.894942208Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.904758536Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:19.904799186Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.024026955Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.03335109Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.03340565Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.03342718Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.03347236Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.03348474Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.033605991Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.033615811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.074691642Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f5eda459-d1e9-44f5-b731-f6490e3fc719/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.082721399Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_f5eda459-d1e9-44f5-b731-f6490e3fc719","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.283770837Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.291509042Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.291550622Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.291557892Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.291564762Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.291568912Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.30136983Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.30140039Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.421434983Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.42938438Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.42941827Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.42942694Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.42944381Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.42945391Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.429597911Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.429605261Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.466690658Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_677df3fa-a1f5-4e63-93ee-592a272d5e3f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.474487914Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:20.665826955Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_677df3fa-a1f5-4e63-93ee-592a272d5e3f","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.675336861Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.683264737Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.683302678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.683310458Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.683315888Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.683320058Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.692568022Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.692593572Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.812358574Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.82024137Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.82028182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.82028833Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.82032171Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.820331681Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.820464391Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.820471791Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.86291241Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_89ffd24b-1dd8-4f52-85d7-6f3c66fce582/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:20.870890497Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_89ffd24b-1dd8-4f52-85d7-6f3c66fce582","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.076675782Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.084653788Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.084688098Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.084715578Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.084723459Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.084774239Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.094383065Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.094429186Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.214015376Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221741541Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221799542Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221806642Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221845902Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221876012Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221973463Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.221981613Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.261940227Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_46b15cce-a870-48ea-aab8-5f90f01b99e4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.270719518Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_46b15cce-a870-48ea-aab8-5f90f01b99e4","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.481484183Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.48954271Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.489609681Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.489617621Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.489624541Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.489630331Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.499137727Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.499170927Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.615174106Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623192224Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623231584Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623239644Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623269024Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623282024Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623410535Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.623427185Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.664670567Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:21.668500919Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Automatically destroying dataset: oxp_0720cae8-ac21-4fb2-9b27-8f836e4c5c41/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.673677399Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_0720cae8-ac21-4fb2-9b27-8f836e4c5c41","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.89909489Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.907225508Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.907257888Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.907262598Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.907266898Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.907271878Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.918232242Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:21.918259852Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.042112398Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.050882859Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.050913199Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.050918249Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.05093387Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.05094163Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.05106512Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.05107055Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.090080019Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.096992189Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.292867547Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.299912858Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.299943289Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.299952929Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.299972749Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.299979289Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.309965467Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.309994357Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.433890313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442591734Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442626954Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442636505Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442647414Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442656375Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442778705Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.442786165Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.482242717Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2c079939-1543-4f7b-ab38-d421cbf95e07/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.490191393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:22.671020853Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_2c079939-1543-4f7b-ab38-d421cbf95e07","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.698914126Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.706918083Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.706976933Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.706986813Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.706996583Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.707005414Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.717095393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.717130803Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.834030678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.842718459Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.842769229Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.842776969Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.842816609Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.842829119Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.8429548Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.84296239Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.882321671Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b6fbc8d5-08ec-4cd5-ae5d-395890772649/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:22.890361888Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_b6fbc8d5-08ec-4cd5-ae5d-395890772649","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.09746358Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105336066Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105368976Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105475667Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105598688Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d05e61ff-a727-4851-80ca-eab919525a00 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6830ec9d-22cb-45c0-ba9d-5a3728bef44f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105660558Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105669508Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.105760869Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.10597306Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.10598806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.10600725Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.10602078Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:891"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106024471Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:896"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106059331Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106067941Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:902"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106112331Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106120741Z","hostname":"buskin","pid":1562,"sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.106206232Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"524bd81d-250f-456c-b856-d78a2e7790ac","remote_addr":"[fd00:1122:3344:101::1]:33471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4869600,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.138055358Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.138163749Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57103"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.13838729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.13839618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.198555702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.198596553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.258901996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.258940476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.317033977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.317079977Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.317093207Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.317150668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.361797009Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.361856719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.361861759Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.415592624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.419708548Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.423913073Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.43538855Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.441260795Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:23.445423919Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/61e07fcc-45b4-4b25-ae0d-e78eee62eb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:23.674164369Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:24.943526937Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:24.943605917Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.422309141Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.422376272Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.909277024Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.909367645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.909403575Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.909494246Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 63.545761ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:25.922757034Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:25.922802084Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 111.603877ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:25.995895472Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 34.57806ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.047543665Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 47.969746ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.054248284Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 145.358228ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.08940471Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 137.204582ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.113569732Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 217.346502ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.242399226Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 240.786735ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.267048651Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 257.547097ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.298711466Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 366.123874ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.479711607Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 383.559119ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.530763056Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 302.465992ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.568258085Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 909.190125ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.854592843Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 565.764198ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.879048146Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 1.153512659s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:26.92237645Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 1.392253659s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:27.456318029Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 513.263998ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:27.774177021Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:27.936354651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4211"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:27.936400152Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:27.943957266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4300"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 872.838512ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:28.085374184Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.091118798Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.091135808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4375"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.135939701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4388"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.135988801Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4252"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.198282986Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4294"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.198320946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4355","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.219957823Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4366"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.219996253Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4408"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 1.038561977s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:28.298843815Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.320292241Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4440"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.320331001Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.437828699Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:62285"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.438055571Z","hostname":"buskin","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"085d8efc-b258-4cae-ae34-45b7e8d875ce","remote_addr":"[fd00:1122:3344:101::2]:62285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.454384766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4487"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.454442717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.487842472Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.520230722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.520334533Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.520774085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.520810945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.556563315Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605524242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605585612Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605594742Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605603282Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605609902Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605616622Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605641742Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605650443Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605657433Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605665253Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605672233Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605678653Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605685133Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605693483Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605700373Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605708493Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605713063Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605717893Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605726493Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605731333Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605736253Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605740853Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:28.605745413Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 615.59652ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:28.85770619Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 1.214967533s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:28.967452333Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 628.303053ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:29.344512651Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 1.332949303s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:29.481689785Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003\") failed: Property not found. retry in 991.329076ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:29.982618619Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 729.867972ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:30.19258887Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4\") failed: Property not found. retry in 1.028685102s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:30.82409163Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703\") failed: Property not found. retry in 758.219575ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:30.931302118Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871008513Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_21f5073c-362d-497d-ae9e-f6ab85374703"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871055944Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_42b0964a-b1a3-443e-b006-c18e1ad8d003"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871066854Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_9704e6fa-a599-4b20-92b9-bc19ae282ee4"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871075484Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871419016Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.871747968Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4887cabd-9cf2-4063-876c-c089d45ae797","remote_addr":"[fd00:1122:3344:101::1]:57103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8733506,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.873566299Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.930410872Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:31.985789666Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.038870497Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.038884397Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.071177186Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.07195003Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.071969321Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.101223832Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.101314943Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52480"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.101492104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.101501524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.121078669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.121126949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.138053318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.138084418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.154987967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.155047997Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.155182388Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.155274319Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.166033772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.198767014Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.199621169Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.199631069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.229643934Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 220, 173]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.229659555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.230076127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.230081197Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.230343769Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.230760651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.25275816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.270403693Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.903702634Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:32.903783204Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 60.81027ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:33.031970465Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 134.412399ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:33.117906079Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 226.876859ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:33.278612191Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 498.367216ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:33.514841635Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 1.101270167s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:34.024794391Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 944.547908ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:35.136438345Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 966.565671ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:36.089624969Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea\") failed: Property not found. retry in 942.041728ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:37.065932259Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.027567434Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.027625494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.028001026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.028358938Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c7f3e36b-57c0-4da6-8e5a-f5f0b8358d5e","remote_addr":"[fd00:1122:3344:101::1]:52480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5927006,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.030542361Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.060670468Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.060831449Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48727"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:38.070755887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.070861077Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ceda5413-e0c0-4e31-a3c5-d6326b134a11","remote_addr":"[fd00:1122:3344:101::1]:48727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9927,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:38.072428826Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.230922595Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:38.231048266Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54306"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:42.240008043Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.240210435Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"2f61ba36-28b8-4528-9203-757c67f8640e","remote_addr":"[fd00:1122:3344:101::1]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009016,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:42.243548954Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.422772194Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.422910895Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38334"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:42.431897348Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.432035439Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6c20c0a6-01da-419a-a80e-e0f77d453e44","remote_addr":"[fd00:1122:3344:101::1]:38334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8960,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:42.433432527Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.637420642Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.637552253Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42238"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_30c18989-6f47-40e1-ab15-11c78531e2ea': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-09T17:49:42.646356214Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.646521485Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"fb3b1f3f-e9e6-448d-9463-f7d6f78875dd","remote_addr":"[fd00:1122:3344:101::1]:42238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8875,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:42.648426587Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.873496975Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:42.873636146Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33798"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.163586758Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ef36abe1-aafd-40f4-aaa1-c00efe7c538d","remote_addr":"[fd00:1122:3344:101::1]:33798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4021751,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3494071282, ip_addr: 208.67.75.242, stratum: 3, ref_time: 1725904185.9544878, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.165880422Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.165889632Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1266"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.19623974Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.19633468Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36776"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.196572732Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.196583952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.215805374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.215855925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.234462924Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.234510004Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.252541879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.252582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.316740466Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.316777876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.381738396Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.381790127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.443080776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.443121026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.504258174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.504298554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.574470406Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.574523506Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.574674007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.574772618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.606713125Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.606758185Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.606766455Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.606788475Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.606798135Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.606923716Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"63ab3736-faff-4ad6-b493-7190e218a1d5","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized)]","latency_us":410531,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.607149277Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"136.95336ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63ab3736-faff-4ad6-b493-7190e218a1d5\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63ab3736-faff-4ad6-b493-7190e218a1d5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.745413157Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.74584193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.7458541Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.763698694Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.763755875Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.781098406Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.781146747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.798576949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.798621209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.816843716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.816890366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.83465065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.83468847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.850544593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.850593854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.8687204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.8687635Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.887051737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.887107428Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.887276349Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.887401889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.918952104Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.918988834Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.918995604Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.919004584Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.919011694Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.919123575Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d9b93661-f54c-48a5-9a74-1b4ee14dd11b","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":173533,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:46.919268356Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"64.626552ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9b93661-f54c-48a5-9a74-1b4ee14dd11b\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d9b93661-f54c-48a5-9a74-1b4ee14dd11b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.984419358Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.984752149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:46.98476735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.003904862Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.003958312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.023467066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.023506057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.042120266Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.042165846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.060044891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.060084171Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.077749174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.077787915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.095979511Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.096021892Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.11445654Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.1145046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.133000298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.133081889Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.13325874Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.13335536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.164812455Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.164856385Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.164866585Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.164876175Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.164886235Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.164999726Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a21eace3-fe11-4956-8072-ee1b060dff60","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized)]","latency_us":180437,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.165178697Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"195.256725ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a21eace3-fe11-4956-8072-ee1b060dff60\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a21eace3-fe11-4956-8072-ee1b060dff60\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.361775748Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.362069339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.362080599Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.38093637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.38097963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.398134891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.398174971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.416382478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.416431318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.434300403Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.434338033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.452048597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.452099137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.470941267Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.470981488Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.489361135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.489404485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.507573912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.507654452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.507917924Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.508077985Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540004132Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540046792Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540057212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540065272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540078622Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.540212623Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"73e568f2-2175-4bd8-a8c2-1446fd33e761","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized)]","latency_us":178305,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.540352224Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"223.717741ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73e568f2-2175-4bd8-a8c2-1446fd33e761\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"73e568f2-2175-4bd8-a8c2-1446fd33e761\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.765756065Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.765980276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.765994436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.784783576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.784839847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.802757561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.802791632Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.821653622Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.821697722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.83840124Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.83844517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.855263269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.855301759Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.873153524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.873191804Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.891103849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.891141869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.907707916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.907755887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.907940908Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.908056508Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.937692122Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.937738332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.937747752Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.937755972Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.937765183Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:47.937859603Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cd93707a-2345-473e-8f2d-d8e4d2246e54","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":172013,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:47.938018524Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"122.558609ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd93707a-2345-473e-8f2d-d8e4d2246e54\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cd93707a-2345-473e-8f2d-d8e4d2246e54\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.061344609Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.061702689Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.061756671Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.080211103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.080258953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.097683505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.097724661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.115762056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.115796716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.133960383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.133999373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.15068644Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.150727915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.166611491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.166653763Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.183876202Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.183917293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.19980742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.199853227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.200022418Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.200129172Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.231324507Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.23136Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.231386354Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.231396685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.231403436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.231474836Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9d9e39ec-beb2-4f30-92e8-120dd02dcb97","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized)]","latency_us":169535,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.231692822Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"273.131022ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d9e39ec-beb2-4f30-92e8-120dd02dcb97\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d9e39ec-beb2-4f30-92e8-120dd02dcb97\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.506827987Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.507139539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.507150347Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.52509798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.525159096Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.54275692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.542792846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.55993998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.559975989Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.579231302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.579264105Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.596059101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.59609481Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.614427612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.61446544Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.631988095Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.632019258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.651645934Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.651761539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.651958631Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.652052775Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.681666212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.681702728Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.681712366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.681721554Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.681745554Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.681914198Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e4b49af6-4222-4aed-8862-a0643b3fa692","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized)]","latency_us":174434,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:48.682088357Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"184.610391ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e4b49af6-4222-4aed-8862-a0643b3fa692\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e4b49af6-4222-4aed-8862-a0643b3fa692\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.867749264Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.86810125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.868120544Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.886928558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.886961811Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.904356931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.904457042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.922526204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.922564127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.940370469Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.94040136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.958146952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.958183243Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.976365502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.976402383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.994140703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:48.994175588Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.010871938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.010905959Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.011075067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.011174189Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041410315Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041447057Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041456329Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041464766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041473691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.041585077Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"822efac4-43de-49ca-a32b-914cbbb4b688","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized)]","latency_us":173210,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.041806117Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"320.533635ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"822efac4-43de-49ca-a32b-914cbbb4b688\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"822efac4-43de-49ca-a32b-914cbbb4b688\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.364030808Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.364239674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.364245028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.382466299Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.3825011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.400064015Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.400099884Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.417643551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.41768571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.435003044Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.435041106Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.453583056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.453620969Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.470752902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.470790458Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.488312465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.488353208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.505678907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.505723889Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.50588843Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.505983052Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.536412368Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.536445762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.536456571Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.536465412Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.536474206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.536549807Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6a3d1d5d-8b17-47a4-a56f-b8bc0d91e49c","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized)]","latency_us":171949,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.5366774Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"380.449319ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a3d1d5d-8b17-47a4-a56f-b8bc0d91e49c\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a3d1d5d-8b17-47a4-a56f-b8bc0d91e49c\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:49.841014912Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"344.417447ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.919362423Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.919733441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.919743519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.93847682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.938509801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.95650747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.956545354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.974421632Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.974460651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.991551632Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:49.991590558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.008750959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.008791356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.026683407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.026721321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.046553256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.046591489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.065295646Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.065344303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.0654972Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.065678095Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.096996701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.09703138Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.097039265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.09704533Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.097052708Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.097130118Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5d13c058-fb90-490c-99e7-1466ff9f0b66","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":177141,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.097245216Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"333.717325ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d13c058-fb90-490c-99e7-1466ff9f0b66\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d13c058-fb90-490c-99e7-1466ff9f0b66\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.433385441Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.433750334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.433761884Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.451626374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.45166859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.470470118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.470511584Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.487919045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.487958439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.505921444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.505958157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.524875344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.524916849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.542681815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.542713896Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.561351936Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.561396534Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.579462361Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.579510249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.579651193Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.579746566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611272166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611311054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611315835Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611321498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611326279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:50.611419195Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cc1d7722-67aa-44bb-90c6-eb0b147f49c4","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized)]","latency_us":177386,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:50.611560393Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"817.222426ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc1d7722-67aa-44bb-90c6-eb0b147f49c4\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cc1d7722-67aa-44bb-90c6-eb0b147f49c4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.431655964Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.432894948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.432900395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.453479473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.453510054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.473235836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.473271471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.491032865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.491060625Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.509154116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.509180187Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.528758712Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.528785262Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.547414449Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.547445189Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.565247357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.565270842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.584794156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.58482817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.585007074Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.585113471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.618958477Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.618999887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.619008578Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.619014916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.619023213Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:51.61910876Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2f57b058-4994-4ad2-80c6-4291c857f2f6","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized)]","latency_us":186795,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:51.619261836Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"646.173527ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f57b058-4994-4ad2-80c6-4291c857f2f6\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2f57b058-4994-4ad2-80c6-4291c857f2f6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.268612257Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.268914171Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.268926583Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.289079374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.289139365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.307879773Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.307917948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.325522528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.3255591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.343790682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.343829288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.362270292Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.362323729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.380281254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.380312707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.398874278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.398916157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.417259749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.417305396Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.417467772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.417561139Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449656201Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449691676Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449699186Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449716895Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449725877Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:52.449804261Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"899c1886-41d8-45f4-9ea8-b37877c48799","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized)]","latency_us":180604,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:52.449942393Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.248289798s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"899c1886-41d8-45f4-9ea8-b37877c48799\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"899c1886-41d8-45f4-9ea8-b37877c48799\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.702726534Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.703849544Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.703856631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.722798742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.722838398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.74075835Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.74079451Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.759086731Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.759127372Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.777951367Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.777986149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.795656639Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.795696679Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.813735478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.813777196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.832514444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.832550659Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.850227964Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.850265493Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.850474716Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.850618905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882538994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882589019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882596567Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882605342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882613761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:53.882694883Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3e542c35-6478-443b-9d84-726782fb38f0","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized)]","latency_us":179377,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:53.882916525Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"906.786587ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e542c35-6478-443b-9d84-726782fb38f0\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e542c35-6478-443b-9d84-726782fb38f0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.793014418Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.793404899Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.793413749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.811567846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.811622354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.828856042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.828905271Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.845802109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.845844241Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.864453826Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.864557565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.881932558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.881967339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.89903065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.899063594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.916432285Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.916470855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.933983777Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.934028862Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.934191097Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.934276606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.965902872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.965935Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.965942481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.965948322Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.96595541Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:54.966068913Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d228329b-4227-492f-8583-1c485e80b25b","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":172392,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:54.966223039Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.584378427s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d228329b-4227-492f-8583-1c485e80b25b\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d228329b-4227-492f-8583-1c485e80b25b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.556051432Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.55642876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.556447623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.578245946Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.578275805Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.597564205Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.597594336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.616398527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.616493408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.635233421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.635272515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.654795884Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.654830047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.673552138Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.673572575Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.693044604Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.69306132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.710995275Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.711043772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.711230326Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.71133361Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744011119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744040931Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744047138Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744054573Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744061267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:56.744165179Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"579c16f2-f5d1-434d-95ff-13bcff371332","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized)]","latency_us":187384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:56.744309011Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.900136351s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"579c16f2-f5d1-434d-95ff-13bcff371332\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"579c16f2-f5d1-434d-95ff-13bcff371332\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:57.41010989Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:57.410284838Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e31c054e-75a9-4ccf-8be7-15255c9bf1ef","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59472","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":73,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.651039219Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.651364796Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.651375643Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.672233729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.672258657Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.691042642Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.691070514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.708426319Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.708453947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.724516917Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.724537946Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.741988037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.742014643Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.758701385Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.758804289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.777359864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.777380808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.795270125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.795297162Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.795456238Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.795540313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.827078274Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.827114237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.82712319Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.827132519Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.827140909Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:58.82722986Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"74ac4d8c-71c8-427c-9195-111614685adc","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized)]","latency_us":175508,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:58.827374161Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"942.631949ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74ac4d8c-71c8-427c-9195-111614685adc\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74ac4d8c-71c8-427c-9195-111614685adc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.774669945Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.775085836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.775112959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.796070951Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.796105357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.813030784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.813066859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.831634803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.831666594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.849502986Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.84953038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.867918277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.867950152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.885257455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.885287914Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.903011221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.903037753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.919839418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.919907153Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.920049729Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.920221963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950000344Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950041051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950049723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950060026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950068464Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:49:59.950153926Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a6d9d81a-52c0-497a-9afc-f94b1ecb46a5","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":174813,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:49:59.950363645Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.966178175s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6d9d81a-52c0-497a-9afc-f94b1ecb46a5\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:49:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6d9d81a-52c0-497a-9afc-f94b1ecb46a5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.923417307Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.923631603Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.923648871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.946161369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.946190873Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.965203808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.965232927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.983263224Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:01.983290646Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.001181558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.001202576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.022078579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.022110116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.041454321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.04147969Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.059014067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.059036595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.077820799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.077864684Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.078060575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.078252997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109021618Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109054216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109060524Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109065353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109069937Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:02.109171872Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"882c6d71-1a1b-459b-a798-8a7a98404e5c","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":185197,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:02.109379811Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.305091316s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"882c6d71-1a1b-459b-a798-8a7a98404e5c\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"882c6d71-1a1b-459b-a798-8a7a98404e5c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.422319755Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.422666547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.422680085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.443845594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.44387085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.462889961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.462911768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.482899937Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.482925193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.500958853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.500987148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.518306013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.518340475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.537464493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.537486365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.555172858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.555192658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.574111221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.574138221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.574319215Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.574426176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606059693Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606087828Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606097062Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606103981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606130672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:04.606252969Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"139838bc-39ab-43fb-91d1-68153d7a783f","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":183309,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:04.606397795Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.733733282s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"139838bc-39ab-43fb-91d1-68153d7a783f\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"139838bc-39ab-43fb-91d1-68153d7a783f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:05.263901299Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"498.746277ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.354495397Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.354829665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.354841674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.376547352Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.376596364Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.396901922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.39692686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.415110401Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.415135611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.43339523Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.433424555Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.452770069Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.45279706Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.470486791Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.470514232Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.488491132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.488514822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.507670686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.507705983Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.507896765Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.508000051Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539457584Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539491981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539499097Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539503644Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539590209Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:09.539707876Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2dc411ae-43ca-4819-8102-20d8686a0a99","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized)]","latency_us":184532,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:09.539874217Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.583487947s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2dc411ae-43ca-4819-8102-20d8686a0a99\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2dc411ae-43ca-4819-8102-20d8686a0a99\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.137289149Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.137812119Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.137822741Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.159046001Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.159073104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.179184549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.179217174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.197948538Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.197977085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.217041777Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.217064099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.235302903Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.235336249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.253888239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.253920555Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.272346125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.272378272Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.290398933Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.290439236Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.290614231Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.290707682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.32339492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.323421629Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.32342807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.323434942Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.323443014Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:14.323547592Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5d05f671-c9b9-4df4-a361-407a4545f752","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized)]","latency_us":185472,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:14.323679283Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"6.447033041s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d05f671-c9b9-4df4-a361-407a4545f752\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d05f671-c9b9-4df4-a361-407a4545f752\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.789355357Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.789816838Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.789826026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.810930808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.810965833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.830599697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.830623669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.845766621Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"884.643346ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.849607747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.84961295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.867532878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.867557225Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.885474546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.885493118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.903706758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.903728566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.921025111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.921046739Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.93896552Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.93899792Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.939239674Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.939341027Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972029042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972078495Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972086792Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972094808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972101511Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:20.972161014Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2961c412-2375-428f-917d-18102a529c97","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized)]","latency_us":182130,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:20.972372193Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"8.156944835s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2961c412-2375-428f-917d-18102a529c97\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2961c412-2375-428f-917d-18102a529c97\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:27.52579989Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55263"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:27.525883966Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3180a94a-9c10-4f11-aa66-7ed27f8944a5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55263","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.153445215Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.153724298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.153735285Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.174320457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.174360049Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.193531381Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.193557284Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.211795046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.211818793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.230184097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.230205528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.249033412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.249050765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.26696295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.266983763Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.284702514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.28472464Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.302168184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.302201035Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.30240093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.302514452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.335284444Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.335315475Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.335324137Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.335334619Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.335341481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:29.335437941Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"650bc940-ec8f-4037-beb4-7ae648dfa264","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\", TimeNotSynchronized), (\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\", TimeNotSynchronized), (\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\", TimeNotSynchronized), (\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\", TimeNotSynchronized), (\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\", TimeNotSynchronized)]","latency_us":181356,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:29.33557371Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"9.211943116s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"650bc940-ec8f-4037-beb4-7ae648dfa264\", \"content-length\": \"124\", \"date\": \"Mon, 09 Sep 2024 17:50:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"650bc940-ec8f-4037-beb4-7ae648dfa264\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:36.81229223Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.83086631s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.574485447Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.574691435Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.574701035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.59448342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.594530923Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.616354813Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.616424422Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.636299816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.636342406Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.655398808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.65543674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.674363553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.674406397Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.692769807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.692807907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.712125008Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.712167759Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.729896274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.729938546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.730093975Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.730195479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.814611025Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.814649276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.814654058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.814659926Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.814664876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.906618835Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.91086081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.914802608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.919034703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.923749829Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.928631225Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.933478521Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.938130811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.942728514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:38.947441647Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.93265719Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.932793832Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.938753462Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.938887169Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.965484849Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.965675488Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.996998729Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:41.997102952Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:42.003910914Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:42.004008396Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 49.302509ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.080214029Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 57.815922ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.088363223Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 70.24075ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.10150944Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 48.638587ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.130229949Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 116.821199ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.139649843Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 25.634824ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.149294697Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 87.631845ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.313756219Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 94.850453ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.516394399Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 147.909561ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.554843988Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 101.605271ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.783679312Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 66.996066ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:42.865016452Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 79.269717ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.044648667Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 245.523866ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.102804193Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 434.115625ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.27571292Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 496.020062ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.424787037Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 249.36093ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.638520152Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 167.748379ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.786141092Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 295.279012ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:43.873078819Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 1.058247901s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:44.560990155Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.033585426s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:44.921427558Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 547.820702ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:44.987199037Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 561.029504ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.135051105Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 598.694597ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.148716485Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 476.109288ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.547985996Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 730.117125ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.634712816Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 658.887503ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.707162518Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 507.773353ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.759687748Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 895.965157ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:45.97888945Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.26380491s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.034823781Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 693.915916ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.277476242Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 595.20053ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.376479208Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 653.616286ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.384547147Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 920.926533ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.886898379Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 756.052837ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.981189777Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 1.155622191s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:46.988277214Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.040455682s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:47.048894212Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 590.525681ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:47.311476905Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.194732639s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:47.749103536Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.259831144s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:47.823154074Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 989.985282ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:47.915373405Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.317840101s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:48.103886498Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 717.848738ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:48.162442234Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 836.154703ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:48.891384227Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.457735533s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:48.917103547Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 774.478371ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:48.955415137Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.075180221s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:49.094845177Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.145571407s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:49.434579893Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 1.084367419s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:49.738703578Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.400509048s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:49.746196414Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 823.405027ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:50.180685906Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 927.473468ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:50.388683444Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 812.430528ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:50.593902903Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b\") failed: Property not found. retry in 1.385335453s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:50.836870267Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 976.717814ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:51.016204093Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.169391537s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:51.16009237Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.068963422s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:51.328763417Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 554.951379ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:51.418196444Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.481126633s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:51.98535733Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.375901808s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:52.005521361Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.069090324s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:52.342358832Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.174138228s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:52.409922274Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 882.840439ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:53.395969988Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.243447409s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:53.424514454Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 774.879477ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:53.481231616Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 662.216567ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:53.598929096Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:53.743695634Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.833768988s"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 928.246131ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:54.267306332Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 940.306282ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:54.27554025Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.227046159s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:54.287785302Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.447762007s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:54.679878859Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.097268416s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:55.20583862Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.315284023s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:55.225747687Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 749.466829ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:55.524010124Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 757.105718ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.13620434Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 535.197718ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.282572598Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 908.341563ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.312338112Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 851.903035ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.550030835Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.223482145s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.826158653Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.396334715s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:56.90240438Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.499320394s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:57.228740281Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 950.83993ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:57.410510326Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:57.630921558Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58260"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:50:57.631015658Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"474812b0-cef7-4b88-b849-1727e6a999ab","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58260","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 975.241872ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:58.057858289Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.081739443s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:58.30718509Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.09370894s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:58.369772087Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 924.373409ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:58.736613029Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 1.449416437s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:59.042985231Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.366710314s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:59.398143971Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 507.345427ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:59.471958733Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 1.428033894s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:59.669936293Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.456986828s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:50:59.987895006Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 577.866251ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:00.501380654Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b\") failed: Property not found. retry in 1.01290118s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:00.773209477Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48\") failed: Property not found. retry in 616.289914ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:01.088129122Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554\") failed: Property not found. retry in 785.083018ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:01.107028393Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2\") failed: Property not found. retry in 1.14768946s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:01.45326179Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634809653Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_14dd30ae-7b79-425c-93c7-fe02960c834b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634885383Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_67aa186b-3457-4f4c-a272-6d6f7b31af3b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634890673Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_f48d5fbc-b227-4070-9b8c-1daa5dd4af48"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634897224Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_83a77c6e-f99a-4321-9983-b522202ce9a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634901724Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_3409b0d3-ece0-4389-9b68-d2f49d8e8554"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.634953134Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.635383366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.635885309Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1a291c2b-714a-48d0-9876-3b61c3a0d195","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":24021736,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.674121263Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52137"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:02.71105641Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3686"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:15.587181227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3704"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.370169639Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3727"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.370729372Z","hostname":"buskin","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"10250563-7473-42e7-a9f0-35232a0931e7","remote_addr":"[fd00:1122:3344:101::1]:52137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":24696063,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:27.378562318Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"9.321320575s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21f5073c-362d-497d-ae9e-f6ab85374703, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 30c18989-6f47-40e1-ab15-11c78531e2ea, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 18779e04-84b0-48e6-a8b5-146b8922c987, kind: Service { id: 30c18989-6f47-40e1-ab15-11c78531e2ea }, name: Name(\"ntp-30c18989-6f47-40e1-ab15-11c78531e2ea\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 173])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14dd30ae-7b79-425c-93c7-fe02960c834b, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83a77c6e-f99a-4321-9983-b522202ce9a2, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9711344c-11c4-490b-9560-e9c007eb0cf3, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 07c1bbdd-f080-4837-9528-41234be47bda, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ca022d43-3da4-4507-941c-6fd9da5b668d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fd75f244-5e91-4f63-b3b2-ae4139e91823, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fe9e311d-41c3-4a6e-8e3e-3fbb54dee639, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e017ec87-832c-4092-8516-57b23da956bc, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: ef3a2fd8-3786-4198-8c2d-a9442073f2de, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bebbab47-f19c-4831-a25e-344318d56393, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 4a391a53-62d4-4c58-a8df-837ca0cab6d9, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 88cec213-8d41-4e46-aea3-2d037677a065, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2ce49a1d-0f2c-40bf-9a41-d49a169c3abb, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External } } } }, OmicronZoneConfig { id: a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External } } } }, OmicronZoneConfig { id: c52a5621-100d-4082-b483-9a9b71016544, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External } } } }, OmicronZoneConfig { id: e37b75a6-6c18-4702-a94d-ecb36cc45c07, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 18d54208-fbd9-4a9a-bb4c-0dda56eeac06, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d731515-3696-45fe-816a-2c2bf0393b4f, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4f04eb1-0d05-41b6-b3d2-1d063e4e900e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External } } } }, OmicronZoneConfig { id: 899a683c-bd60-4b15-b12a-e5db4452f148, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.412094075Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.412808169Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59416"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.41472163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.41472893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 42b0964a-b1a3-443e-b006-c18e1ad8d003, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.440103178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.440144318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9704e6fa-a599-4b20-92b9-bc19ae282ee4, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.460846569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.46089801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 21f5073c-362d-497d-ae9e-f6ab85374703, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.481894553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.481939723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f48d5fbc-b227-4070-9b8c-1daa5dd4af48, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.502448323Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.502494673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 14dd30ae-7b79-425c-93c7-fe02960c834b, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.52234911Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.52239286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3409b0d3-ece0-4389-9b68-d2f49d8e8554, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.542014415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.542050595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 67aa186b-3457-4f4c-a272-6d6f7b31af3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.560705264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.560747495Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 83a77c6e-f99a-4321-9983-b522202ce9a2, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9711344c-11c4-490b-9560-e9c007eb0cf3, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.583098766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.583137216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9711344c-11c4-490b-9560-e9c007eb0cf3, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.66923954Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50551"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.669908824Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"af334c33-f0c7-4ca4-9d8b-9c9ac6ee8dd3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50551","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":513,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07c1bbdd-f080-4837-9528-41234be47bda, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.705468993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.705507463Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 07c1bbdd-f080-4837-9528-41234be47bda, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e017ec87-832c-4092-8516-57b23da956bc, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.829230998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.829287218Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e017ec87-832c-4092-8516-57b23da956bc, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 88cec213-8d41-4e46-aea3-2d037677a065, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.966778354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:27.966823104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 88cec213-8d41-4e46-aea3-2d037677a065, dataset_name: DatasetName { pool_name: ZpoolName { id: e68e093e-0654-4278-a79f-9c234ab4cc35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ce49a1d-0f2c-40bf-9a41-d49a169c3abb, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.091921197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.091972537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 2ce49a1d-0f2c-40bf-9a41-d49a169c3abb, dataset_name: DatasetName { pool_name: ZpoolName { id: f5eda459-d1e9-44f5-b731-f6490e3fc719 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.238856108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.238905868Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03, dataset_name: DatasetName { pool_name: ZpoolName { id: 677df3fa-a1f5-4e63-93ee-592a272d5e3f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c52a5621-100d-4082-b483-9a9b71016544, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.371971428Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.372031698Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c52a5621-100d-4082-b483-9a9b71016544, dataset_name: DatasetName { pool_name: ZpoolName { id: 89ffd24b-1dd8-4f52-85d7-6f3c66fce582 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37b75a6-6c18-4702-a94d-ecb36cc45c07, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.47820155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.47825474Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e37b75a6-6c18-4702-a94d-ecb36cc45c07, dataset_name: DatasetName { pool_name: ZpoolName { id: 46b15cce-a870-48ea-aab8-5f90f01b99e4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18d54208-fbd9-4a9a-bb4c-0dda56eeac06, dataset_name: DatasetName { pool_name: ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.607604168Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.607647808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 18d54208-fbd9-4a9a-bb4c-0dda56eeac06, dataset_name: DatasetName { pool_name: ZpoolName { id: 0720cae8-ac21-4fb2-9b27-8f836e4c5c41 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d731515-3696-45fe-816a-2c2bf0393b4f, dataset_name: DatasetName { pool_name: ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.736332742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.736375803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0d731515-3696-45fe-816a-2c2bf0393b4f, dataset_name: DatasetName { pool_name: ZpoolName { id: f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4f04eb1-0d05-41b6-b3d2-1d063e4e900e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.867830293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.867879883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a4f04eb1-0d05-41b6-b3d2-1d063e4e900e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2c079939-1543-4f7b-ab38-d421cbf95e07 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 899a683c-bd60-4b15-b12a-e5db4452f148, dataset_name: DatasetName { pool_name: ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.992400533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:28.992464463Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 899a683c-bd60-4b15-b12a-e5db4452f148, dataset_name: DatasetName { pool_name: ZpoolName { id: b6fbc8d5-08ec-4cd5-ae5d-395890772649 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.106969894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.107143185Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.107451357Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.107874689Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.25310781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.267335044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.267511815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.281502487Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.281742238Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.281778728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.30598508Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.306205102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.330048331Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.330085791Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.470637624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.470810625Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520659407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520709678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520713938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520716958Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520719818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520722638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.520725488Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.521621793Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.528588734Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.534090216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.541204258Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.547839506Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.553310219Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.713695349Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.718870209Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.723428426Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.72759491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.731673224Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.736301781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.741120119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.746519841Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.751121048Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.755884626Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.760789005Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.765504882Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.771409317Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.776982009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.783677328Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.82653861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.826688651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.832671686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.838001747Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.842562044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.847518593Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.852663353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.858077675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.900081281Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:29.965505564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.020113764Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.110412143Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.11497838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.11499645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.169345299Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 187, 44]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.169379959Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.170946888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.170964068Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.256404259Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 223, 94]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.256452159Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.257858647Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.257874817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.314098047Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 139, 64]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.314120037Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.315596625Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.315613606Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.383398683Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 185, 185]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.383429243Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.384866522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.384883022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.441125001Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 201, 216]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.441160001Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.44609379Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.44611218Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446238271Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446252381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446303141Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446309931Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446328141Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446333981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446456442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.446464282Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.450426685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.451809923Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.505053246Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.508734327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.511999666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.512081907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.512093877Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.515182295Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.518476404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.518490804Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.742549297Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.754270886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.766391757Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.779598884Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.793922708Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.807459298Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.815934267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.82670934Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.837955446Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:30.849624274Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.396508238Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.396636929Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.396701479Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.423085384Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.429901973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.430134055Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.477714804Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.477899475Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.485197108Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.485403619Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.48558551Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.48572802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.485894092Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.486857337Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.486888297Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.493535526Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.498879718Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.504540801Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.510754197Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.516618912Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.538404649Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.53855721Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.538619701Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.545591811Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.560929451Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:33.561097652Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 53.145241ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.882078743Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 52.441238ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.892574925Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 58.965826ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.902541563Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 74.993893ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.911952228Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 46.579608ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.921509614Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 64.686543ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.929649482Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 28.413728ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.937451478Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 46.894415ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.947679267Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 32.25378ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:33.962320583Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 25.355303ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:34.147943831Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 41.70191ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:34.180110449Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 118.640259ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:34.610432201Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 69.277483ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:34.979931856Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 72.122894ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.07265Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 144.326285ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.363945417Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 60.592723ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.55046873Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 54.44899ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.617559493Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 146.398689ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.670709344Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 119.382296ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.703188525Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 113.166299ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.729440998Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 101.18497ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.754193403Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 103.188356ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.779421031Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 90.031172ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.810413983Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 74.137032ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.831714047Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 123.537519ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.852223118Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 96.483482ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.867223106Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 99.769987ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.884491017Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 279.902945ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.899317504Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 287.359228ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.913475487Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 162.087464ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.927227077Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 222.677075ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.939173207Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 299.410149ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.952662596Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 119.531106ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.967239852Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 292.636063ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.982147229Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 190.158483ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:35.99595673Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 217.744599ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.008689965Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 199.828304ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.021961252Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 548.962575ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.035415981Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 138.589592ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.04880393Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 165.506183ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.06077732Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 522.89946ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.104834938Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 303.767423ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.12052303Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 540.915104ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.183225567Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 379.04853ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.198264615Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 511.367552ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.214098088Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 347.080948ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.227863009Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 480.38722ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.242552925Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 597.869753ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.255755902Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 506.265386ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.269417192Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 252.64454ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.283798597Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 431.615666ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.297024124Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 250.507262ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.307088333Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 842.790387ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.438449782Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 419.684878ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.551377134Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 425.721169ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.571091219Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 886.320145ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.589189105Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 879.526991ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.604409214Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.065567352s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.619546343Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 657.864053ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.647397906Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.099289644s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.917957482Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 594.6043ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.931064669Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 549.226871ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.945323052Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 882.894581ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.959403985Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.057640786s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.972844604Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 753.64198ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:36.988752947Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 720.843828ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.001721393Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 806.781405ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.017411965Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.110461696s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.303222159Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 801.267618ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.316818339Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 648.235367ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.493497674Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.447115968s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.505157452Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.372580481s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.514182195Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.120238751s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.538742599Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.037362126s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.698785867Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.433609005s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.737100561Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.433804416s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.754101831Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.302632922s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.83585154Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.273830328s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:37.854230407Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.399311863s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.029913857Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 991.148017ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.043377396Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 852.186501ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.130253885Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 678.753556ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.153830313Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 856.572897ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.428528912Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 668.252858ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.676097203Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 771.015324ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.752230769Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.798852312Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.798931722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.799047533Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.807841924Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.816480875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:38.816662576Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 606.94306ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.84919729Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.053485744s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.902979612Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.297794818s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.970459617Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 817.408931ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:38.998695123Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.179422414s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.057293566Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 34.872816ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.086845269Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 46.970258ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.096311225Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 42.421901ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.106502385Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 75.229862ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.132484797Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.316237026s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.142378245Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.496551104s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.152583355Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 123.916531ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.161373776Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 148.126644ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.1705565Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.443943552s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.186502653Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.017134934s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.199974252Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 210.098978ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.217423264Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 602.54197ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.296345397Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 296.740295ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.308983121Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 125.928971ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.333856797Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.382709342s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.356254548Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 587.20827ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.441125204Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.48015602s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.451348974Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.251640375s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.468335553Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 388.292463ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.479042916Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.049774216s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.536318792Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 287.086005ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.617399137Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 877.667803ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.839572789Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 576.281966ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.893316443Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.031250043s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.925435602Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 542.090686ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.948919919Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.286318902s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:39.982968729Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 458.131037ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.044724711Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 878.276178ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.243426825Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 903.521922ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.269071665Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.022879195s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.296439645Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.457790947s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.472778549Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 986.331453ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.487387484Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 727.54632ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.504416384Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 656.472438ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.516272413Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 716.640775ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.599956324Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.163457164s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.642494533Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:40.642849685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:40.643145657Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:40.643289808Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:40.649211182Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 562.543234ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.666170592Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.283659882s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.731491894Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 791.716135ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.745514787Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 699.203219ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.757036594Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 40.209819ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.860774062Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 33.599207ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.870889021Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 110.679051ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.912595266Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 124.684576ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.921186946Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.443618913s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.941959327Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 661.761608ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:40.969416018Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 249.446063ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.033491714Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 210.167964ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.057159422Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.345805026s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.13357426Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 796.106192ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.183831535Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.440535764s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.196409508Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 536.197439ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.240511567Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.201275474s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.252482337Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 432.328227ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.281031444Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 831.458021ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.29389232Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 245.29092ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.306826345Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.332576204s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.329201406Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.486865765s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.339148015Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.186413357s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.467581807Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.016247135s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.486468208Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.371992652s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.550280442Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.015984742s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.563469509Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.048938677s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.642437571Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 618.212961ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.728809068Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.116059926s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.791014252Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 554.931487ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.828321181Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.227634703s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.943546036Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.431698183s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:41.991824829Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.30753458s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.028674165Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.443536484s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.139570504Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 925.291351ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.361088102Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.308916056s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.395371313Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 720.368645ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.406372587Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.276008207s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.467806167Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 946.215415ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.494183662Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.392406559s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.515418096Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.450554763s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.59278251Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.49063752s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.649864034Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.172163154s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.666807323Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 801.823076ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.678520222Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.431541523s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.704391713Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.370573243s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.836645658Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.123341314s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.918085816Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 968.67842ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:42.93258973Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.434830822s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.141130232Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.329886958s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.183765952Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.416197456s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.301588423Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 939.06358ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.350399369Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.14217941s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.437360408Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.297183669s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.453567873Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 911.398111ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.493589777Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.022259566s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.593061Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.07654003s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.714722223Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.460796279s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.762164451Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 575.011052ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.85239771Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 535.85692ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.93433257Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.481764425s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:43.950270653Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 667.567247ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.052480532Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.215558291s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.061657365Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.063270791s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.145528087Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 705.515404ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.153349043Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.239364001s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.214875503Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.095075654s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.301418Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 550.245282ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.417157418Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 895.383023ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.440177612Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 762.915905ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.481647705Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.427527426s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.525899075Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 673.29731ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.590286812Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 518.261258ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.602125821Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 632.151032ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.627814722Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.341715898s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.72998084Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.217893353s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.740655263Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 980.717863ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.759912086Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.160612432s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.800627144Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 884.18341ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.868678803Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 570.026457ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:44.97743619Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 931.016988ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.129798853Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.224228462s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.218330612Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 916.479063ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.232441184Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 856.445655ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.253524788Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 803.025062ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.269086069Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 517.018915ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.277121206Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.296621843s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.285848607Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 797.387575ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.344170029Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.009450294s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.405522929Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.286869672s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.440406393Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 629.594566ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.4620356Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.40725105s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.556206731Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 583.546116ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.749313063Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 842.720693ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.761242413Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.135541126s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.803245059Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.471634626s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.962677253Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 643.918449ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.970420178Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.12371746s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:45.978077983Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 652.218945ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.071195909Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.407014631s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.080533743Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.479511053s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.088816642Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.358877499s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.101071604Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.397206038s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.119635062Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 921.817704ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.151316748Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.381478552s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.159303775Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 878.816016ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.341499802Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.216853993s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.423195931Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 875.315783ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.451715268Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 988.925629ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.592887605Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 673.670842ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.613346695Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 818.448227ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.623786996Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.117961865s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.733395478Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.024774269s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.741292515Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.100995993s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.947929075Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.30567412s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:46.974080889Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.469464743s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.081050366Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.464571858s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.11087607Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.428259433s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.230409761Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.363782464s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.2951715Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 784.662364ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.334848003Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.245289802s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.477933701Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.000765116s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.486740012Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.45370398s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.494907Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.183457627s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.503823352Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.065134349s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.526111113Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 589.829728ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.549592851Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 878.728984ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.577428804Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.000980568s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.5904662Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.144424713s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.649416456Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 983.045139ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.774877581Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 682.76403ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:47.86182017Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.453138492s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.058437222Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.376160661s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.12973752Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 970.517628ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.149381675Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 882.634632ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.288707501Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.426370815s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.465607138Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 571.000677ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.497280983Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.098055091s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.553733544Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 951.696308ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.561692101Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.388081749s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.585184019Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.141720208s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.599681033Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 651.927963ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.607758381Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.464809532s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.667841143Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.190873262s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.675764659Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.367547498s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.696798142Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.267397116s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.731771087Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 715.00987ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.766745072Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.025838525s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.803034675Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 774.006801ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:48.958387495Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.15862267s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.118174841Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:49.120162303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.178367503s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.17766332Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:49.1777248Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.245638461s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.191100569Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 911.352538ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.269404057Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 36.889571ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.383786228Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 123.164038ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.430722382Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.418477919s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.490917395Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.377659201s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.515998152Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.285192642s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.524341481Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.124694517s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.531893265Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 242.968913ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.562378034Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 730.862239ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.660956681Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.16464199s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.742000076Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 753.267117ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.751381041Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 537.832171ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.814671552Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 598.893114ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.839667838Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 664.531863ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.889698722Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 527.836385ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:49.921137796Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 609.933448ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.000915073Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.150984508s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.018247285Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 583.349222ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.075018657Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.424971645s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.144630625Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 772.957299ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.191604211Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 683.05629ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.2871866Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.093114039s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.362883064Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 740.496737ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.372650981Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 639.114076ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.401719922Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.139898547s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.446593125Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 896.371416ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.455272125Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 907.739865ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.463021521Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.070406174s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.513445006Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.170633837s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.562991296Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.458599562s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.620768845Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.409060052s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.666278882Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 871.54477ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.674181908Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 948.159712ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.819944292Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 757.561537ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.90497216Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 519.241469ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.917753255Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 840.884541ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.926463846Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 952.131112ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.975731355Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.177355221s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:50.9851105Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.376659797s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.050831445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 946.596074ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.123765002Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 922.546942ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.180810956Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.183524246s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.364483263Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.284775607s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.383534924Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.213004552s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.448812576Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 624.775805ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.466447529Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.163773874s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.557391492Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.470376152s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.58104226Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 806.65473ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.595967418Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.222099958s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.607897668Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 580.968171ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.675250852Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 899.416607ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.745756696Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 593.326835ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.777460451Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 589.363785ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.786180253Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.90523837Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"13.321839016s"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.223420416s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:51.939977914Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 813.01476ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.085063564Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 554.574619ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.097756918Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.234514351s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.109519597Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 985.91062ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.120867674Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.113019471s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.132017469Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 743.09984ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.176297448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 771.652263ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.270644381Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 872.096749ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.383192231Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.229605257s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.394617047Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 750.543287ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.416480835Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.460547011s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.441610113Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 880.137535ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.560744811Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.111051692s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.658510473Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.243515848s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.668615543Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.250858133s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.678839453Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.016092139s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.687369703Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.466424884s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.732781189Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 572.412174ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.843298116Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 934.239321ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.912425221Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 715.21702ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:52.933869337Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.003490458s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.057389041Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 972.715643ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.069794383Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.482380221s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.120229139Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 942.843885ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.179540986Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.29610942s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.193625699Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.452311086s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.259012602Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.464348256s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.271929078Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 699.046176ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.358073632Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.457635322s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.427953372Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 833.264699ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.454018804Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 525.573406ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.639055049Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.381171288s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.661108968Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.139514104s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.71788146Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 837.778632ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.786999375Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.488021547s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.860134884Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 769.777378ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.912311459Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 969.833856ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.924026168Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.243944904s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:53.943003869Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 808.268956ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.055301777Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 732.287885ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.068561705Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.048967252s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.076826013Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 828.205503ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.135244646Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 590.690838ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.1769959Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 674.301099ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.211581573Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.202307373s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.299700629Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 571.79789ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.503815665Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 686.87844ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.612508312Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.38753995s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.635620987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 751.732027ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.694454342Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.008349264s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.723486832Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 806.602026ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.747743004Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 951.177319ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.779592951Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.077116966s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.813283868Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.469371636s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.883753621Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.276979825s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.89889797Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 660.94128ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.913013173Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 869.260452ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.923864016Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.49427488s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.93302506Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.195397363s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:54.977036558Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 847.936825ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.05584953Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 922.171435ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.088956113Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544\") failed: Property not found. retry in 1.493448445s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.13785992Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.13184171s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.198540606Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 517.880104ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.314481865Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.000720785s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.35981584Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393\") failed: Property not found. retry in 1.280797197s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.459394404Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 906.820533ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.514555077Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.284302789s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.56797497Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.051981989s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.585125171Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 919.903141ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.74380852Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 563.203542ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.754570153Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.205955988s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.804546056Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.05826215s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.845578937Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 669.335813ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.905020735Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.000134546s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:55.918847326Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 881.701726ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.026597937Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 514.605145ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.042135068Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 1.166160504s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.189513252Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.148657641s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.20281735Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 560.944783ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.332162557Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 1.333663281s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.34796987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.249871302s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.36677648Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.047102597s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.383522598Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 1.262473247s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.434219175Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 876.173275ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.449745926Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 850.31421ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.573788853Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.038432955s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.59211532Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 860.402308ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.667587143Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.054114271s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.68077911Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639\") failed: Property not found. retry in 1.432183553s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.873855531Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.006497335s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.913823775Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.438743674s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.930932845Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 673.298657ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.94541817Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 1.185629549s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:56.964775804Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.086963538s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.026945778Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de\") failed: Property not found. retry in 1.422306737s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.339388939Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8\") failed: Property not found. retry in 1.423786761s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.37031999Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 968.116691ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.382709743Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.122135877s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.441354916Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 661.349001ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.454178331Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 1.168187917s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.544879713Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 535.52757ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.628458642Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb\") failed: Property not found. retry in 1.030856842s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.643146238Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03\") failed: Property not found. retry in 1.369058632s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.659132932Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e\") failed: Property not found. retry in 610.350581ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.699386758Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f\") failed: Property not found. retry in 511.331268ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.715989615Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:57.726751108Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:51:57.729301823Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"dd86d688-4c95-45b4-88f2-fff9b602f06b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38191","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1329,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 1.314391934s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.750750349Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 1.218069302s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:57.93346651Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07\") failed: Property not found. retry in 1.08508415s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.127827498Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 897.493148ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.143797312Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9\") failed: Property not found. retry in 631.633742ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.166119133Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06\") failed: Property not found. retry in 1.436862085s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.183950497Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 504.369415ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.364693356Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148\") failed: Property not found. retry in 1.42019473s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.382845482Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065\") failed: Property not found. retry in 1.026979257s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.576082415Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda\") failed: Property not found. retry in 902.624786ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.732529981Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3\") failed: Property not found. retry in 675.253077ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:58.888513625Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823\") failed: Property not found. retry in 1.366039973s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:59.067750485Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d\") failed: Property not found. retry in 787.683123ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:59.088480627Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc\") failed: Property not found. retry in 903.778023ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:51:59.170650668Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486236196Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_07c1bbdd-f080-4837-9528-41234be47bda"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486303556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_899a683c-bd60-4b15-b12a-e5db4452f148"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486311807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_fd75f244-5e91-4f63-b3b2-ae4139e91823"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486318687Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_oximeter_fe9e311d-41c3-4a6e-8e3e-3fbb54dee639"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486324656Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_c52a5621-100d-4082-b483-9a9b71016544"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486330667Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_ca022d43-3da4-4507-941c-6fd9da5b668d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486336417Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_e37b75a6-6c18-4702-a94d-ecb36cc45c07"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486343017Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_9711344c-11c4-490b-9560-e9c007eb0cf3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486349727Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_clickhouse_e017ec87-832c-4092-8516-57b23da956bc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486355957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_18d54208-fbd9-4a9a-bb4c-0dda56eeac06"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486362497Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_0d731515-3696-45fe-816a-2c2bf0393b4f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486378147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_4c6ef9f1-4001-4c65-ae99-bafe915b66f8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486383757Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_4a391a53-62d4-4c58-a8df-837ca0cab6d9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486389867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_a4f04eb1-0d05-41b6-b3d2-1d063e4e900e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486395807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_bebbab47-f19c-4831-a25e-344318d56393"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486402057Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_88cec213-8d41-4e46-aea3-2d037677a065"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486408527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_ef3a2fd8-3786-4198-8c2d-a9442073f2de"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486788199Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_2ce49a1d-0f2c-40bf-9a41-d49a169c3abb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.486801019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_a6fd4f40-6a6d-4e01-a5a7-0e34b5f5ae03"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.489807327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.496859558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.505188617Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9a469a76-416e-4383-aa32-659685becd61","remote_addr":"[fd00:1122:3344:101::1]:59416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":33091398,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.518253064Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1286"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.518408115Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.518658816Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.525443306Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.528170822Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n local_pref: None,\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.594572051Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:867"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.613915964Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.668755025Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.83334231Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:00.945787379Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:01.179315547Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:01.369775473Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:01.657339987Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:01.913774589Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:02.252235012Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:02.497194948Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:02.763737929Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:03.461353147Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:04.172929296Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:52:05.265413937Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"26.381685121s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:05.451796719Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:06.96585786Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:07.79214973Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:09.319941242Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:10.819079495Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:12.627137627Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:16.497580693Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/1068bc7e-e3ab-44f6-b097-299854b0081f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:19.392898846Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7bf4982b-676a-4f39-a13a-42c9f45b95d7\", \"content-length\": \"173\", \"date\": \"Mon, 09 Sep 2024 17:52:22 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"1068bc7e-e3ab-44f6-b097-299854b0081f\\\"\", request_id: \"7bf4982b-676a-4f39-a13a-42c9f45b95d7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:22.265646857Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:27.766975957Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64845"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:27.767838092Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c5887270-4a30-41af-ac42-9d446360eff8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64845","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":620,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c5d09c0d-7435-4394-b97e-9d4c49d8ef7e\", \"content-length\": \"173\", \"date\": \"Mon, 09 Sep 2024 17:52:29 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"61e07fcc-45b4-4b25-ae0d-e78eee62eb60\\\"\", request_id: \"c5d09c0d-7435-4394-b97e-9d4c49d8ef7e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:29.675567379Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3762bf19-6e73-4127-ac27-2075056b81e9\", \"content-length\": \"173\", \"date\": \"Mon, 09 Sep 2024 17:52:34 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"61e07fcc-45b4-4b25-ae0d-e78eee62eb60\\\"\", request_id: \"3762bf19-6e73-4127-ac27-2075056b81e9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:34.125108648Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ce3454ed-b17f-4516-9246-53b32f21fc3a\", \"content-length\": \"173\", \"date\": \"Mon, 09 Sep 2024 17:52:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"61e07fcc-45b4-4b25-ae0d-e78eee62eb60\\\"\", request_id: \"ce3454ed-b17f-4516-9246-53b32f21fc3a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:44.25551968Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.524083508Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.613899874Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40202"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.617865227Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.706497026Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"b8206ce8-5f79-45dc-a84e-1b19b4de4809","remote_addr":"[fd00:1122:3344:101::a]:40202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92452,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.709047351Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.709121952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.720666819Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:49.721313043Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"90c5717e-7e3f-40a9-88d8-55fc8072d767","remote_addr":"[fd00:1122:3344:101::a]:40202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":13633,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.607697992Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.610298537Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a809104e-4bea-4165-b23a-dce4633a0251","remote_addr":"[fd00:1122:3344:101::c]:54498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2577,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.613325075Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37747"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.613408725Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"dd35b4b7-76cc-489f-b6c2-120802ad3eec","remote_addr":"[fd00:1122:3344:101::b]:37747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.659025983Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49696"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.6636649Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b165ecd7-9979-4e52-8037-e367c5aabc72","remote_addr":"[fd00:1122:3344:101::b]:49696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4588,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.739293853Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0adc7c06-5137-493e-8b76-45de23ec0236","remote_addr":"[fd00:1122:3344:101::b]:37747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12825,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.742507442Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42419"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.742646363Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9623d4ac-a627-43d8-87a8-af2425da9763","remote_addr":"[fd00:1122:3344:101::c]:42419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.753231585Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bdbe4b58-a7ae-420d-8ed1-583cc663e864","remote_addr":"[fd00:1122:3344:101::c]:54498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":77,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.940057829Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.94025795Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"bd3d35f8-1fd5-4546-bfdb-999731fdbaf3","remote_addr":"[fd00:1122:3344:101::a]:52440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.951555796Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.951644987Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fda4aeca-1197-4909-90c3-b6dd15d78316","remote_addr":"[fd00:1122:3344:101::a]:63440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:53.962625432Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"157b50c4-cd30-4c06-92b1-8d1bf7a70f3e","remote_addr":"[fd00:1122:3344:101::a]:63440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.789010193Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64563"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794070353Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794363254Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794641406Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794662326Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794675316Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794689126Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794694886Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.794719536Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"521d2f0e-6512-419f-bea2-29a91399f4c3","remote_addr":"[fd00:1122:3344:101::c]:64563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5667,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.795860853Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:939"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.98998568Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45364"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:54.990094991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.079153803Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"14d7aed0-1daa-40cf-a2ce-72fd2c5532e6","remote_addr":"[fd00:1122:3344:101::c]:45364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89077,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.081499057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.081569287Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.081938789Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.082380292Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"41aac253-afae-4548-9506-50648ebec531","remote_addr":"[fd00:1122:3344:101::c]:45364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":995,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377554511Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39528"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377834173Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377880813Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377912713Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377941924Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377960494Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.377972774Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.378001854Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.378019384Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bd402fbd-c9ac-43a6-b03e-e1c21c4a8cdc","remote_addr":"[fd00:1122:3344:101::b]:39528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":377,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.38924379Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44802"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389462161Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389490711Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389506151Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389519512Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389534201Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389546572Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389562242Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.389573742Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"18dac836-493f-402c-a8e9-60b9b5795154","remote_addr":"[fd00:1122:3344:101::c]:44802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":213,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.657905844Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56987"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658165695Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658210876Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658241596Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658266256Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658276466Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658298786Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658350516Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:55.658356267Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"04a521ef-bac9-40ad-9039-25a92712c39d","remote_addr":"[fd00:1122:3344:101::a]:56987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":311,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:57.802985891Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41797"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:57.803057821Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"5b8e9468-9bed-4a76-af3b-be71519755dc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41797","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:59.778232643Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:59.778341694Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fa57e783-cd69-434a-993e-4ac66f7bcb56","remote_addr":"[fd00:1122:3344:101::c]:43106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:52:59.788110081Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f9ce0af2-8936-4e67-ad20-515332f841c6","remote_addr":"[fd00:1122:3344:101::c]:43106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:01.789807741Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:47540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:01.831995865Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"a081cdd6-253e-4509-8760-337136bd44d9","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39496,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.728792081Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39803"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.728922122Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e4ff4ea1-25be-4916-8634-9521735d3984","remote_addr":"[fd00:1122:3344:101::c]:39803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.741803667Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1113c43d-a049-4b50-a6a7-2f6007fd6ec3","remote_addr":"[fd00:1122:3344:101::c]:39803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.799402045Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.799694697Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"372aa1bf-423a-44f3-a7c4-5e73a54ccd24","remote_addr":"[fd00:1122:3344:101::b]:48346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":222,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.802071971Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.802173151Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5b50f4e7-b05b-4c1a-b5cc-4aa369fe6590","remote_addr":"[fd00:1122:3344:101::b]:47389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.822308629Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5b48e1e3-d04b-49e1-b882-98aefd653cb5","remote_addr":"[fd00:1122:3344:101::b]:47389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1761,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.955278238Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:23.955479659Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5de360cf-9aba-4676-82e2-3669c49d4430","remote_addr":"[fd00:1122:3344:101::c]:63413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.05972944Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49462"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.063587153Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.13823043Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.204537818Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5437fdfb-0755-41c9-a4b7-2d0055a31502","remote_addr":"[fd00:1122:3344:101::c]:49462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144748,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.20478424Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56193"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.20480078Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37158"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.20484228Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.205788016Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9c41ef6c-f605-4e33-a18b-0bfa04664b2d","remote_addr":"[fd00:1122:3344:101::a]:56193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":949,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.205802376Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.209487778Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"203a2d6d-5c33-4801-9ec1-34a2b699cfeb","remote_addr":"[fd00:1122:3344:101::a]:37158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4523,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.215634394Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"93ff8dc5-a216-4d9a-92af-9b8608be69ed","remote_addr":"[fd00:1122:3344:101::c]:49462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10680,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.216280607Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5b268211-be3a-4c24-be55-6d5ccb9301ae","remote_addr":"[fd00:1122:3344:101::a]:56193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":778,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.251761305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.268107421Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/d05e61ff-a727-4851-80ca-eab919525a00/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.269142637Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.270932608Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.271858233Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.271930693Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"9efd80c9-b54b-41e5-bde7-ece5a525018a","remote_addr":"[fd00:1122:3344:101::c]:49462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":11415,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.313607478Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"64fd0c68-c613-4025-8352-e4f0b1f7a637","remote_addr":"[fd00:1122:3344:101::b]:46988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108631,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.314312292Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"cdd9b6c9-f2e8-4347-88b3-298e0ec8d5be","remote_addr":"[fd00:1122:3344:101::b]:46988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":408,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.360548563Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64715"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.360650853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.39760647Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.457933432Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"60c463a0-346e-401e-9572-f1b3f3722f50","remote_addr":"[fd00:1122:3344:101::a]:64715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97335,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:24.458795977Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c6961f5b-4110-48a0-9381-ad6e35b2cfa3","remote_addr":"[fd00:1122:3344:101::a]:64715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":664,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:27.836355616Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:27.836443887Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"37e58b2f-240d-449c-ae46-a259cb04c19f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40161","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:31.794412675Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"e55ea249-124a-4357-95f6-86e21aee611e","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7050,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.588086287Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.588240658Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a4712d80-772a-4da2-8d4d-67533952b802","remote_addr":"[fd00:1122:3344:101::b]:57817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.60064772Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"06dfc1cb-001d-4896-9331-9fc785354d8b","remote_addr":"[fd00:1122:3344:101::b]:57817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1427,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.681748135Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41737"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.681827676Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b4ff9b69-9c25-4264-bfdf-60c4fe86869d","remote_addr":"[fd00:1122:3344:101::c]:41737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.691462652Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7ce7c922-950f-4568-816e-09a73fba22f9","remote_addr":"[fd00:1122:3344:101::c]:41737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.734862577Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.735014228Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"240414b8-45cb-4520-868e-251f673de6fc","remote_addr":"[fd00:1122:3344:101::b]:63112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.757051997Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.757200898Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"dc16c25b-0978-41d2-95f5-2f0dddcb1abc","remote_addr":"[fd00:1122:3344:101::c]:50664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.799332175Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34299"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.800918444Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.851281199Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.92148308Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1bd4e904-0d28-40aa-88e7-7331788a3ad3","remote_addr":"[fd00:1122:3344:101::b]:34299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":122074,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.921628581Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48538"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.921753822Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:53.964535123Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.036407644Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8c216c13-8442-4562-9327-9a349f034d00","remote_addr":"[fd00:1122:3344:101::c]:48538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114696,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.036691425Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.036896866Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f09cb747-7e8b-4e9e-a5e4-22e6c90c93de","remote_addr":"[fd00:1122:3344:101::a]:53151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.036984887Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7d63bb87-b4b8-404a-a76a-5653a54c5e8c","remote_addr":"[fd00:1122:3344:101::b]:34299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":244,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.036995027Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a024a6a3-4b61-4f03-b954-cea9f15d3e28","remote_addr":"[fd00:1122:3344:101::c]:48538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":255,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.047377798Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6c2dacce-d057-406d-8f11-035f38810a34","remote_addr":"[fd00:1122:3344:101::a]:53151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":142,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.119929973Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.120130334Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"83045ef0-ed3a-4e23-a75d-5913fd1306f0","remote_addr":"[fd00:1122:3344:101::a]:45224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.148824172Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54496"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.148971383Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.191484492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.247045448Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7010bc56-6ab8-4bc1-a072-259e07964ead","remote_addr":"[fd00:1122:3344:101::a]:54496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98159,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:54.247926293Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7b5a97b4-0863-47e0-8bd3-416565dca14b","remote_addr":"[fd00:1122:3344:101::a]:54496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":695,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:57.869461559Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:53:57.869534169Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"43e05087-fd24-4347-bd73-ec4fe6e3a63b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40288","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:01.794654425Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"5a05a5d0-7417-4a9c-b938-2a958f7373d0","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6597,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:22.555987598Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.719407064Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47267"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.720650771Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1669fc29-c413-4b62-bcde-924ed44e5ebc","remote_addr":"[fd00:1122:3344:101::b]:47267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.84366678Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44779"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.843825865Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e1c7c8d6-2291-4dbf-b8b0-a31140b8a253","remote_addr":"[fd00:1122:3344:101::b]:44779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.858151117Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b173853e-831b-4403-979b-f99f41173232","remote_addr":"[fd00:1122:3344:101::b]:44779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":171,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.876463154Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54221"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.876564884Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"da2f1713-cef7-495c-bf25-1ebed9da8b2d","remote_addr":"[fd00:1122:3344:101::c]:54221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.956471833Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.956716594Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"87dd53cb-667b-4a10-b982-3ad37260196b","remote_addr":"[fd00:1122:3344:101::c]:42970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.963387233Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"64471f78-d4cc-4010-84df-0853fc9b6d93","remote_addr":"[fd00:1122:3344:101::c]:54221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.986235297Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63699"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:23.986321668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.029317809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.104919812Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f3f1ad21-77ae-4aa5-bb94-b37c3172b7bc","remote_addr":"[fd00:1122:3344:101::b]:63699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118623,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.105283384Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63121"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.105354505Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.105442875Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"245bb3eb-fa2f-4068-955c-7b47c0088570","remote_addr":"[fd00:1122:3344:101::b]:63699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":224,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.105570966Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f9465567-8898-4416-bac2-6874ec7c35a5","remote_addr":"[fd00:1122:3344:101::a]:63121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":157,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.105696527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.148564248Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.182526837Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.186137458Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4bdd6742-5e66-4257-b423-01304261aca7","remote_addr":"[fd00:1122:3344:101::a]:47340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":640,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.197907597Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5ba31cf1-be6b-4423-a474-46d4428e7f17","remote_addr":"[fd00:1122:3344:101::a]:63121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":564,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.206603008Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a6fc0a77-bc38-4a76-a85d-c5f4e1e95bde","remote_addr":"[fd00:1122:3344:101::c]:64912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101049,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.20702205Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5cf7fa60-fbde-4df6-b93f-cd8bc52c37d7","remote_addr":"[fd00:1122:3344:101::c]:64912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.219985806Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52715"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.220131967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.261997852Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.319137957Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"66a4f8e4-5d66-463c-9163-658255c00ec2","remote_addr":"[fd00:1122:3344:101::a]:52715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99066,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:24.319485557Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a2b44de9-0bab-4737-a672-0b3d52f533a1","remote_addr":"[fd00:1122:3344:101::a]:52715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:27.902869113Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:27.902971194Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"24e1433a-412f-4e02-bf84-2f7432f9913c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61305","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:31.795879731Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"5dc7821f-0750-4e4d-aca7-d9840655dd65","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6561,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.636355793Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.636559464Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2e129638-2333-4696-8703-168863e90aea","remote_addr":"[fd00:1122:3344:101::b]:62014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.646191021Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37779"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.646302851Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5c08ccfd-8a69-47ab-84e1-9f9f45ef9efb","remote_addr":"[fd00:1122:3344:101::c]:37779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.655340184Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"adb6fea2-764c-4adb-abf9-82ecd1c369c4","remote_addr":"[fd00:1122:3344:101::c]:37779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.655648236Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60013"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.656568392Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1a27fe7d-c7c7-41f2-b240-5e63f37e0c39","remote_addr":"[fd00:1122:3344:101::b]:60013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":886,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.667354175Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"846f162d-27f2-43df-915a-861efcd9f639","remote_addr":"[fd00:1122:3344:101::b]:60013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.747224123Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63096"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.747814646Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.793046491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.871677312Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8cbbe9c6-cb08-4df1-a700-0ea7d296e510","remote_addr":"[fd00:1122:3344:101::b]:63096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124390,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.872058164Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"191534b1-d67f-4a7b-b315-bb6e532318f2","remote_addr":"[fd00:1122:3344:101::b]:63096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.901070014Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.901328075Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4e53ec1b-16dc-4c5d-be24-bd76fa2fc975","remote_addr":"[fd00:1122:3344:101::c]:46570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.931304621Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55199"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.931456872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:53.969217173Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.027395714Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"35a91d40-00da-4603-82f1-c9f136e3d767","remote_addr":"[fd00:1122:3344:101::c]:55199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96002,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.027570455Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52066"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.027598125Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.027811707Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f727efb4-c292-40e3-b8cc-87850baecfa4","remote_addr":"[fd00:1122:3344:101::a]:62835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.028007918Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2d8b5dfc-cbce-429b-91bf-601e9cf3203e","remote_addr":"[fd00:1122:3344:101::a]:52066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":292,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.028091478Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9cf5ed0e-f9e3-4cc7-ac95-a79be3bb1b3b","remote_addr":"[fd00:1122:3344:101::c]:55199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":265,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.035833944Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"fe22a831-0ba1-4cbf-9196-30c32c0a4102","remote_addr":"[fd00:1122:3344:101::a]:62835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.047323191Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57231"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.047429691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.083608634Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.136809025Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ced738ca-5488-4d5e-9dcc-2fb8b9ad5c99","remote_addr":"[fd00:1122:3344:101::a]:57231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89428,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:54.137237978Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"226a84a9-e384-4fae-9f95-c2de9078cc2f","remote_addr":"[fd00:1122:3344:101::a]:57231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:57.937866514Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:54:57.937931444Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"9130651d-8d01-428e-a7a7-6d492badc589","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36152","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:01.797523875Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"c218c44e-ba44-4406-8c4e-936237ed86bd","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7868,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.661650467Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.661813028Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c75e713c-9010-4ab2-9fff-cbed21c34ab9","remote_addr":"[fd00:1122:3344:101::b]:43294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.683916628Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5ce676d3-9481-4654-b2c6-9b61ed6dc6d6","remote_addr":"[fd00:1122:3344:101::b]:43294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1414,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.756408453Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52949"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.756503123Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7461d1bc-d2f0-4fc0-bf16-90eb7ed9170d","remote_addr":"[fd00:1122:3344:101::c]:52949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.770152433Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"21637a23-8dda-4e0c-94fc-7fbbe508a31e","remote_addr":"[fd00:1122:3344:101::c]:52949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1429,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.903156394Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.903356552Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"7e0d7d10-b115-4c52-a09c-3f8c3fba18a0","remote_addr":"[fd00:1122:3344:101::c]:54503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.914834331Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.915003612Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"35728636-9548-43d3-856d-48b73b3d0895","remote_addr":"[fd00:1122:3344:101::b]:40904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.940696892Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43296"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.941603608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:23.983870715Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.043921547Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3a72e51b-a23e-45c8-9a97-29b3817cf100","remote_addr":"[fd00:1122:3344:101::c]:43296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103158,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.044069988Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39800"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.044193369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.076464498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.132151164Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"db4cd447-c646-4440-8fe6-989796eaabb9","remote_addr":"[fd00:1122:3344:101::b]:39800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88002,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.132485056Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.132758408Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f9a56da7-aad0-4b13-8af9-900e3c347fee","remote_addr":"[fd00:1122:3344:101::c]:43296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":284,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.133558882Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"435066ae-4ffa-4ace-be80-c721cb8b5dfe","remote_addr":"[fd00:1122:3344:101::a]:52925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":968,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.133615883Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"64d20417-2793-42e4-81c1-d422ff118722","remote_addr":"[fd00:1122:3344:101::b]:39800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.241944127Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.242103958Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"00a59e98-9389-42f1-aa62-ae01f4842da5","remote_addr":"[fd00:1122:3344:101::a]:56467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.252071277Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e4ba353f-f38f-42f3-9648-7968a9c6e5aa","remote_addr":"[fd00:1122:3344:101::a]:52925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.330154014Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46446"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.330231845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.366457427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.423561651Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0d8ce8c0-0540-432d-805f-982a925b6864","remote_addr":"[fd00:1122:3344:101::a]:46446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93356,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:24.423868723Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d06c2a23-caff-4ba2-8b3b-d275b7dc173e","remote_addr":"[fd00:1122:3344:101::a]:46446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:27.971783149Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58304"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:27.971861389Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"4aea744e-b343-4940-a122-8e322ad32083","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58304","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:31.887283058Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"6ad06fed-9c20-4642-b688-6cc0849233bd","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8385,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.564054266Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.564195236Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7b411243-03e3-4967-b78a-d3f6aa115c65","remote_addr":"[fd00:1122:3344:101::b]:63510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.579988839Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"848de329-1b83-4358-a6f9-a739cd52f591","remote_addr":"[fd00:1122:3344:101::b]:63510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.687195307Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63839"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.687298598Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c86ca09d-0669-40c8-9017-bdd4228a8d04","remote_addr":"[fd00:1122:3344:101::c]:63839","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.730774043Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.731774318Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d0c76419-8d2b-4723-958b-f9a58bfe915f","remote_addr":"[fd00:1122:3344:101::c]:39487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":972,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.734032282Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2b01e116-5126-4766-a9ff-7f5fb59fc990","remote_addr":"[fd00:1122:3344:101::c]:63839","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.820030846Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35096"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.820144166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.866973281Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.929350056Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d2b23b3e-86cd-4e06-bc3b-3388e7c5bbef","remote_addr":"[fd00:1122:3344:101::b]:35096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.929590877Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62994"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.929672018Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52243"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.929897769Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.93002571Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7b40c8ce-a67a-4ddf-97b3-d57bfbfa9329","remote_addr":"[fd00:1122:3344:101::b]:35096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":293,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.934675837Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e01b0e95-5f1e-4f86-9337-2dd76a33b707","remote_addr":"[fd00:1122:3344:101::b]:52243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4831,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.96081989Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65065"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.960988341Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"639f4766-3efb-4955-88e6-fbdc81adad80","remote_addr":"[fd00:1122:3344:101::a]:65065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.968126713Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:53.974973153Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"06a88910-0ad6-485e-9db2-b081d93fcd40","remote_addr":"[fd00:1122:3344:101::a]:65065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":192,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.035370067Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"eeafeb69-6786-4371-8eec-2d3e91e83eed","remote_addr":"[fd00:1122:3344:101::c]:62994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105648,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.035783759Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b98f6190-8b8a-428e-979a-c0e2a18769b7","remote_addr":"[fd00:1122:3344:101::c]:62994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.053896056Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.054089917Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1f10af57-3912-4d85-ad07-9012c60a0fc7","remote_addr":"[fd00:1122:3344:101::a]:47979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.065579634Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54976"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.065696305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.102666611Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.159760476Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8d449b2f-ceeb-43df-9ab4-a8d3784eec17","remote_addr":"[fd00:1122:3344:101::a]:54976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94114,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:54.160645638Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c1e681f5-0522-4140-9691-f5099747bb57","remote_addr":"[fd00:1122:3344:101::a]:54976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":703,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:58.005442976Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:55:58.005515837Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0e63e5f6-eec4-457a-a979-124c8c874cf4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34073","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:01.887268118Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"1b90511b-699a-47a4-91dd-9b91d10d0759","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8034,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.794143433Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49850"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.795756482Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"830bcf8e-fa49-477f-acfb-b3c7d199bc39","remote_addr":"[fd00:1122:3344:101::c]:49850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.823823897Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"91342dd6-7f80-4ddf-a1b6-7bb37cf7d6bc","remote_addr":"[fd00:1122:3344:101::c]:49850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.896680044Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33108"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.896874235Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a82e2691-dc2f-4c1e-82f9-2039da552c7e","remote_addr":"[fd00:1122:3344:101::b]:33108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.9250872Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57149"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.925254391Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"250c7ba8-444d-43b1-b3f2-e8892d637180","remote_addr":"[fd00:1122:3344:101::c]:57149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.938799961Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.938922231Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"299dc242-133f-4552-b598-ff62da39202b","remote_addr":"[fd00:1122:3344:101::b]:55310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:23.948167975Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c4c5a47a-ea5c-4de8-9af5-31576ce2a959","remote_addr":"[fd00:1122:3344:101::b]:55310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.016440365Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57207"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.016538476Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.062783967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.129201636Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9d9fe229-c53c-4ca4-a4b3-f04be4a6d9c4","remote_addr":"[fd00:1122:3344:101::b]:57207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112683,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.129365417Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48878"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.129382317Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.129541248Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"63d6990c-8e69-40a0-8041-9735a41ab910","remote_addr":"[fd00:1122:3344:101::a]:37115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.130768435Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.130851966Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"203c8122-dde9-496f-ab8b-1e019f73e8b4","remote_addr":"[fd00:1122:3344:101::b]:57207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1311,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.138772832Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2ed24d8d-378a-44b8-8fcd-912e97573331","remote_addr":"[fd00:1122:3344:101::a]:37115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":191,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.169768414Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.226455106Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f722d344-574a-4546-a71d-708cfd6026b0","remote_addr":"[fd00:1122:3344:101::c]:48878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96849,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.226837558Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f2104cc3-dfbd-4a23-b07c-0d37e7832eff","remote_addr":"[fd00:1122:3344:101::c]:48878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.295009428Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.295191249Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"70564603-9ced-4028-b2c5-689332604ade","remote_addr":"[fd00:1122:3344:101::a]:33654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.390806879Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54863"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.390921659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.429933558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.487767266Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"08a9cc1a-379b-4648-b1a4-821b3184c4a3","remote_addr":"[fd00:1122:3344:101::a]:54863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96908,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:24.487995938Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fb82710e-136b-401b-95f8-881299a52b7f","remote_addr":"[fd00:1122:3344:101::a]:54863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:28.038547678Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:28.038644279Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"af58dea2-2f8c-4163-92fd-f1cd02fb3444","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41340","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:31.886389871Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"b41f2922-66a4-45b6-aa51-34f7b120ec87","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6656,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.629939658Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.630113279Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e1c051da-0f08-417e-9539-ac5121b8f526","remote_addr":"[fd00:1122:3344:101::b]:33825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.638905721Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3bba7e48-5fa5-4b7c-bdc6-52300ea13a5b","remote_addr":"[fd00:1122:3344:101::b]:33825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.719558694Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.719654954Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ba07ec1f-76e5-40bf-ab06-b9c483c06706","remote_addr":"[fd00:1122:3344:101::c]:64159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.729631803Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5dc0b579-9a81-4b4f-b726-8722e172236a","remote_addr":"[fd00:1122:3344:101::c]:64159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.759581918Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39564"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.759659598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.80766886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.882516538Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"17701f54-29b4-44ec-99e2-c9d01be59873","remote_addr":"[fd00:1122:3344:101::b]:39564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":122891,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.88273409Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37197"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.88279473Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43294"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.88281902Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63138"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.883018811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.887072625Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"6f091ab5-9b6e-42fa-ba57-68e13cdb0562","remote_addr":"[fd00:1122:3344:101::b]:63138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4024,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.887227056Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c815dfb9-82a1-4ded-8012-34265b53ba48","remote_addr":"[fd00:1122:3344:101::c]:43294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4062,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.887349267Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ac6e563b-5470-45d0-b890-6a8ea75dcc3a","remote_addr":"[fd00:1122:3344:101::b]:39564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4411,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.920836993Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.977498885Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fd0d216f-fdb6-47b4-800c-ef7c7e5620f4","remote_addr":"[fd00:1122:3344:101::c]:37197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94617,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.977907167Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ba50b199-f457-4252-89a4-eba43b8e2563","remote_addr":"[fd00:1122:3344:101::c]:37197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.996350885Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59128"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:53.9971061Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"db371ecc-a438-459e-98fd-5dc7ab398764","remote_addr":"[fd00:1122:3344:101::a]:59128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":721,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.022433248Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.022628859Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"335256e7-60cb-4171-906e-8dadc7268e4c","remote_addr":"[fd00:1122:3344:101::a]:63127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.031193489Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ffb58abd-6aeb-4da7-bf0e-4c104ce6d3aa","remote_addr":"[fd00:1122:3344:101::a]:59128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.160512907Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39384"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.160626808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.199046963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.256313928Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7ad7b99f-784c-494d-86e6-f086a02cd05a","remote_addr":"[fd00:1122:3344:101::a]:39384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95759,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:54.25656319Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"271c9ba3-206b-4dad-94fd-2997be66c77b","remote_addr":"[fd00:1122:3344:101::a]:39384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:58.072500456Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52764"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:56:58.072571517Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"55ca90e8-925c-4284-ae13-17a46b8d47aa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52764","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:01.887092875Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"368eac54-32d4-4bc4-8d43-6f8793fcaced","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.747746036Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.747880497Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"458b9b8d-145b-4955-b2b2-7b906f2c2962","remote_addr":"[fd00:1122:3344:101::b]:43452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.758360808Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ff36bc25-b014-4351-b4b4-e3a672f48c7f","remote_addr":"[fd00:1122:3344:101::b]:43452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1119,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.758453928Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.7586379Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"64e48ebd-2db9-47eb-87e6-4fcf89c56be4","remote_addr":"[fd00:1122:3344:101::c]:65169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.770141247Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9eef1d7c-4164-4c96-a192-747a09f60728","remote_addr":"[fd00:1122:3344:101::c]:65169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.8986538Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57973"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.898861001Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"11a64ab2-20a9-445a-963e-c8310aaa8d7c","remote_addr":"[fd00:1122:3344:101::b]:57973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.961128366Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63025"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:23.961244167Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.003103982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.06427471Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4d4f1951-186c-43da-b7b3-2265c89a9708","remote_addr":"[fd00:1122:3344:101::b]:63025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103079,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.064552332Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.064645433Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64661"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.064659713Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51878"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.064761143Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"86e25953-de8d-4373-a404-9c20de6fa445","remote_addr":"[fd00:1122:3344:101::a]:64534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.064923814Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aefd5927-bb71-48e7-a1b8-2be7d38a6ad1","remote_addr":"[fd00:1122:3344:101::b]:63025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":368,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065081455Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"921b6064-a515-4183-96c4-c32a65799fbc","remote_addr":"[fd00:1122:3344:101::c]:51878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":323,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065112505Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065150665Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065208746Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065232786Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065258106Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065284856Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065305036Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.065333857Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"62e27a2b-416b-4207-a1b5-186b67aac206","remote_addr":"[fd00:1122:3344:101::b]:64661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":552,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.081961274Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43446"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.082239376Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.12392152Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.184358584Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"94cee6fb-438b-43b6-9eb9-a1ba91eb00d7","remote_addr":"[fd00:1122:3344:101::c]:43446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102222,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.184930817Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"19c496dd-a30d-4cb6-81ad-3d6b62add3a0","remote_addr":"[fd00:1122:3344:101::c]:43446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":293,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.195948362Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35725"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.196902947Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.196940428Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.196984558Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.197004128Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.197018128Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.197036358Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.197061158Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.197075628Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"33978667-75f1-419f-92d4-6a13e7cd32a7","remote_addr":"[fd00:1122:3344:101::c]:35725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1064,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.220509006Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.220640647Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"501ae593-002f-4bbb-9c36-c8d1d9b0495b","remote_addr":"[fd00:1122:3344:101::a]:55683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.226465741Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8ae8f7b2-9661-4157-a072-082de0bde9d8","remote_addr":"[fd00:1122:3344:101::a]:64534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.305290762Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65246"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.305395863Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.347011847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.407390541Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"efe4affc-2c2e-47a3-bb22-cb5044551a5c","remote_addr":"[fd00:1122:3344:101::a]:65246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102051,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.407682932Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"eb46dff0-3a59-423a-8f54-a2c4d685a4c2","remote_addr":"[fd00:1122:3344:101::a]:65246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.441688981Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38177"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.441985882Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5db99faa-847d-4c67-baad-c7351bdbf8f8, kind: Service { id: 07c1bbdd-f080-4837-9528-41234be47bda }, name: Name(\"external-dns-07c1bbdd-f080-4837-9528-41234be47bda\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 94])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffc9837e-602d-49be-aea3-1a5919d80850, kind: Service { id: 9711344c-11c4-490b-9560-e9c007eb0cf3 }, name: Name(\"external-dns-9711344c-11c4-490b-9560-e9c007eb0cf3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 216])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2b31f448-70f4-4ab8-9cd0-36ff4c28255b, kind: Service { id: ca022d43-3da4-4507-941c-6fd9da5b668d }, name: Name(\"nexus-ca022d43-3da4-4507-941c-6fd9da5b668d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 185])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 49852929-112c-4ccc-bc3e-c924ce561472, kind: Service { id: fd75f244-5e91-4f63-b3b2-ae4139e91823 }, name: Name(\"nexus-fd75f244-5e91-4f63-b3b2-ae4139e91823\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 64])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f29fffc4-b4e4-4069-af7f-4a0100b36168, kind: Service { id: 4c6ef9f1-4001-4c65-ae99-bafe915b66f8 }, name: Name(\"nexus-4c6ef9f1-4001-4c65-ae99-bafe915b66f8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442036453Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442085253Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442110903Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442146333Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442168513Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442197934Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:24.442213414Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e4391d5c-8309-43f2-98b4-8e43335afc2e","remote_addr":"[fd00:1122:3344:101::a]:38177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":446,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:28.108164282Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:28.108235672Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"fa07c563-ed7a-40f1-8e46-85680862cd27","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55886","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:31.888215269Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"4ecc1b06-d0db-4202-9f21-a0ba8bc58a02","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6585,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.581811531Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.581972592Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a5727053-6f0d-401a-997c-b239de844039","remote_addr":"[fd00:1122:3344:101::b]:63735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.604168912Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.604315933Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"75426916-0081-40cb-9ddc-3d58bcf16bc4","remote_addr":"[fd00:1122:3344:101::c]:38281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.604942526Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d7b648a6-2096-415f-8f35-bc56e1a282c5","remote_addr":"[fd00:1122:3344:101::b]:63735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.615344387Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e2dc21dc-19e4-47b2-af99-464dd6cf5912","remote_addr":"[fd00:1122:3344:101::c]:38281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.756601295Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.756774556Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"728d565e-9616-4f17-93b4-272e4a630e2a","remote_addr":"[fd00:1122:3344:101::b]:61400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.765309556Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.766239082Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"aa03b52b-b935-4e5b-9c78-bea64a2e92b2","remote_addr":"[fd00:1122:3344:101::c]:47763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":879,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.844361609Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51902"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.844561961Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.88719941Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.948821191Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5e842230-03d3-4130-81ed-f6c38647a5ce","remote_addr":"[fd00:1122:3344:101::b]:51902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104401,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.949003092Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62654"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.949194303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.952900265Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b1b11a6e-e06c-4cd8-a389-dee5394318b7","remote_addr":"[fd00:1122:3344:101::b]:51902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3726,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.984891313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.9946105Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:53.994792901Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"42bcac0e-db8a-4974-8750-044af817a5ff","remote_addr":"[fd00:1122:3344:101::a]:47865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.003824434Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"264345cc-9779-4d31-85af-101339951cb8","remote_addr":"[fd00:1122:3344:101::a]:47865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.024080822Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.024526645Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"52f2bb9a-e2cb-402c-b60f-e20bc013cc53","remote_addr":"[fd00:1122:3344:101::a]:58827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":330,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.044402281Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4f873aaf-c6fc-40d4-a65e-2c3b490213c5","remote_addr":"[fd00:1122:3344:101::c]:62654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95308,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.044843584Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"af39b25d-59df-418a-adc6-03215f0cb5c2","remote_addr":"[fd00:1122:3344:101::c]:62654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.141669281Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64272"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.142402875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.183291095Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.240978083Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"46e100a6-a573-497d-a10d-91d2114d4f32","remote_addr":"[fd00:1122:3344:101::a]:64272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99238,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:54.241277315Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"874a5ab1-6913-4b87-9396-7ca4431c0464","remote_addr":"[fd00:1122:3344:101::a]:64272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:58.144374091Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:57:58.144500701Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"488f02ec-f48f-428e-a45c-153e4e6c1dc3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55729","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:01.997693296Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"7beecdcb-991c-4045-bdff-876dba5edcc1","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7077,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.795131586Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32785"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.795319887Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"daa10b92-eba0-4d40-b308-74702638d305","remote_addr":"[fd00:1122:3344:101::b]:32785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":148,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.809019358Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a933f81f-afbf-461a-8866-f9d3fa859594","remote_addr":"[fd00:1122:3344:101::b]:32785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.935920921Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.936013971Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5a504dfe-9fb1-4d66-91e2-45a17812136a","remote_addr":"[fd00:1122:3344:101::c]:50810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":63,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.948612465Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a57e55b9-7aea-4a41-a65b-d36f7616f349","remote_addr":"[fd00:1122:3344:101::c]:50810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.970262872Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46051"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.971103587Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"abb387ba-3ac3-4e92-80fb-ae539822b953","remote_addr":"[fd00:1122:3344:101::b]:46051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":821,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.981377757Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.981511788Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ed5cb17b-9f37-46d4-ae2e-140e390fd378","remote_addr":"[fd00:1122:3344:101::c]:54585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:23.999896206Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62420"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.000818111Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.043130499Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.10300492Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2589b45f-7bee-410e-a4bc-0c7803039ec6","remote_addr":"[fd00:1122:3344:101::b]:62420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103022,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.103203301Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47318"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.103386922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.107090704Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3b0452e1-10da-46f0-99b8-a9599035bb05","remote_addr":"[fd00:1122:3344:101::b]:62420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3734,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.138176916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.191863101Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61609"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.192138492Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4ff3f272-4ea4-449c-a3c6-f16f7400bc3b","remote_addr":"[fd00:1122:3344:101::c]:47318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88864,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.192180282Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0f32585d-897d-4524-9e5c-868739938160","remote_addr":"[fd00:1122:3344:101::a]:61609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":199,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.192484114Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bd22f502-0f70-4af3-aaf9-a77deb637755","remote_addr":"[fd00:1122:3344:101::c]:47318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.199574476Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.199739637Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"8f397d47-c2de-4aed-bef0-e55ff310a439","remote_addr":"[fd00:1122:3344:101::a]:55800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.201062604Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"609c117c-d280-4c47-b88c-31aa2626f315","remote_addr":"[fd00:1122:3344:101::a]:61609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":763,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.322035603Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40757"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.322140024Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.360122956Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.413761081Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a3180c4f-c222-4c86-8bc7-115f19982bf4","remote_addr":"[fd00:1122:3344:101::a]:40757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91678,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:24.414014672Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"43ea2b0f-bf9f-46e9-992f-2a42edb0f000","remote_addr":"[fd00:1122:3344:101::a]:40757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:28.179530512Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36279"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:28.179599883Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ac862bf0-c0de-43bd-9095-11c4e5ecb7d9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36279","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":51,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:32.001148962Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"a6f94761-17cb-4924-9c16-45a05262be61","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7237,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.668065173Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62645"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.668916522Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f04590a6-b7a4-4e95-a7ac-1fdc42f2d61a","remote_addr":"[fd00:1122:3344:101::c]:62645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":887,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.678843926Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7370d927-1cb2-4d40-9d49-b442a8064a6b","remote_addr":"[fd00:1122:3344:101::c]:62645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.685348914Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.685498225Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e20b01bb-8e7a-4455-8dcd-b59dfc4e9ebf","remote_addr":"[fd00:1122:3344:101::b]:42924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.694505618Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6563b99c-2955-457f-913b-4eaf1a4404ee","remote_addr":"[fd00:1122:3344:101::b]:42924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.741626654Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42887"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.741829095Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"18d566e3-9d28-4356-8f2b-3479b6dd37e9","remote_addr":"[fd00:1122:3344:101::c]:42887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.775299651Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39038"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.775391212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.818279403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.878130874Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5d5a2c7a-6013-4a05-aaa8-44bde781639b","remote_addr":"[fd00:1122:3344:101::c]:39038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102780,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.878311165Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53577"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.878389845Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39723"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.878549666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.883007782Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bc5b8ab3-6d3e-4146-be03-c337769a81c3","remote_addr":"[fd00:1122:3344:101::c]:39038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4414,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.883195094Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5f7aade3-e984-41d1-be70-0b470400e007","remote_addr":"[fd00:1122:3344:101::b]:39723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4681,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.913068709Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.916206997Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.916496258Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"245a8807-1d66-4d8a-943c-fc0ad4bbe976","remote_addr":"[fd00:1122:3344:101::a]:42988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":204,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.928536939Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"29e0d2d1-dea0-4629-9983-eed099aa5adf","remote_addr":"[fd00:1122:3344:101::a]:42988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1084,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.973230181Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5bdbb042-4a1f-4ffe-8df4-79af73ad49df","remote_addr":"[fd00:1122:3344:101::b]:53577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94798,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:53.973632403Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2c5dc200-7346-476d-b51e-1f362a8726db","remote_addr":"[fd00:1122:3344:101::b]:53577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.081207394Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.081424815Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"8003b7cb-f8f6-4cbf-94a9-03faaaab2f9b","remote_addr":"[fd00:1122:3344:101::a]:58302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.146645157Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59855"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.146790918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.181785103Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.238433425Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"febd7cb0-86d8-42cb-9a94-5980bdc14791","remote_addr":"[fd00:1122:3344:101::a]:59855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91737,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:54.238730146Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"06324f1d-e1a6-4453-8320-f0731e8eb3b7","remote_addr":"[fd00:1122:3344:101::a]:59855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.217427117Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.218277552Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"cb40750a-ccda-4ead-9bea-c75dd5b968a0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53205","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":801,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.798671053Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34610"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.798826094Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f2f9cae3-0196-479e-9165-e59bd32e17d8","remote_addr":"[fd00:1122:3344:101::a]:34610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.802943258Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(6460566) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.804705898Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"ef3bc583-6862-4edb-bc9e-f438cbb945c6","remote_addr":"[fd00:1122:3344:101::a]:34610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5696,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.968282596Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61451"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.994175588Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:570","metadata":"InstanceMetadata { silo_id: 8dd86d6f-ad2d-45da-852c-8351c18c2cc5, project_id: 7f64605e-b035-408a-9a6a-d8f7d5f3252b }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-09-09T17:58:58.496726Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(760054d9-c981-48b3-88ba-f2e26d33bb6c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-09-09T17:58:58.539917206Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(6460566))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-eaf7a0b5a9d\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3d51e186-2159-48c3-bf34-c47f7cf6e34c, read_only_parent: Some(Volume { block_size: 512, id: 4ef107b1-d950-4fe9-9175-980b859ce150, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d09c4ac9-5522-462d-a22f-856a2751a875, key: Some(\"0AIv4BNGw5iDC/JGKKWaI0ZkTuvWfKNbHYhIEGyFGcQ=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3d51e186-2159-48c3-bf34-c47f7cf6e34c, key: Some(\"s9ODaR7tDM7jnWQ8kxA+DktuC+UtBusc5QVRpfmhDeE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(..) }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.994304919Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.996382166Z","hostname":"buskin","pid":1562,"propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:991","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(6460566))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21680f30-0060-42cd-8e6b-66d832fccd47, kind: Instance { id: 9a69ee61-cd07-4e16-8bce-71a799ee7a11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 208, 226])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6460566), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-eaf7a0b5a9d\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3d51e186-2159-48c3-bf34-c47f7cf6e34c, read_only_parent: Some(Volume { block_size: 512, id: 4ef107b1-d950-4fe9-9175-980b859ce150, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d09c4ac9-5522-462d-a22f-856a2751a875, key: Some(\"0AIv4BNGw5iDC/JGKKWaI0ZkTuvWfKNbHYhIEGyFGcQ=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3d51e186-2159-48c3-bf34-c47f7cf6e34c, key: Some(\"s9ODaR7tDM7jnWQ8kxA+DktuC+UtBusc5QVRpfmhDeE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-09-09T17:58:58.496726Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:58.998747565Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c","method":"PUT","req_id":"cb5ea5d2-751b-49c9-8a9a-a8860c7d1969","remote_addr":"[fd00:1122:3344:101::a]:61451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":30334,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.111771607Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.114391582Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c/state","method":"PUT","req_id":"f8cf17e7-475e-41ef-8f95-e1a1c0d13a65","remote_addr":"[fd00:1122:3344:101::a]:59010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2533,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.12088699Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 208, 226]), slot: 0, vni: Vni { inner: 6460566 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.121031201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.162206683Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:58:59.186847037Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:00.356937092Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:00.357263764Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 33.149749ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:00.552353857Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 130.874487ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:00.594919446Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 210.521036ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:00.743534527Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 320.24397ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:00.969834353Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 413.756949ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:01.307784513Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 1.37847244s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:01.732862654Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:02.001032605Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"8d6f9da7-395d-451e-96bc-499b7b83ac78","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6778,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 705.026674ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:03.123519511Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 1.203464365s","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:03.840731723Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 599.668673ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:05.055391989Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 634.407902ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:05.665626884Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:06.32878983Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1449"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 64.372682ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:06.337782832Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 136.621199ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:06.412353639Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 173.500049ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:06.558760577Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\") failed: Property not found. retry in 405.660709ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:06.741420427Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.158691632Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1458"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.196400773Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1482"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.339826823Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-09-09T17:59:07.343884227Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.343906847Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.344908723Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-09T17:59:07.343884227Z }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.345457766Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-09T17:59:07.343884227Z }, migration_in: None, migration_out: None }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.50664476Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.506761391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-09-09T17:59:07.562006285Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.562092815Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.562174236Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-09-09T17:59:07.562006285Z }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.562223206Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-09-09T17:59:07.562006285Z }, migration_in: None, migration_out: None }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:07.603631299Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999576s","next_cleanup":"Instant { tv_sec: 34669, tv_nsec: 200110218 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:22.568315921Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.743354894Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.743497395Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5237a9ab-3f48-4b2c-b8d0-a0c3aca22d4b","remote_addr":"[fd00:1122:3344:101::b]:60852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.797934284Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.799780945Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c/state","method":"GET","req_id":"46f30c23-53db-4f86-a5a2-3696900f2dc4","remote_addr":"[fd00:1122:3344:101::c]:33619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1796,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.806980977Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.807214798Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c/state","method":"GET","req_id":"731552fe-0f72-47b0-90e4-da8c5cc9c965","remote_addr":"[fd00:1122:3344:101::b]:50525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.832220285Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.832417056Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"61fad09b-4e5b-413b-8771-6c07d89dde3d","remote_addr":"[fd00:1122:3344:101::c]:48191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.949952415Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.950138636Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"dcafbee7-7dbf-4385-aed0-00060a18740b","remote_addr":"[fd00:1122:3344:101::b]:42867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.959776472Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.959912333Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c/state","method":"GET","req_id":"61d3e709-b068-4dcf-aada-f8a71e7792ea","remote_addr":"[fd00:1122:3344:101::a]:49150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:23.977286445Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"43e07ffd-6e55-45f2-8027-86f1d66f86ea","remote_addr":"[fd00:1122:3344:101::b]:60852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":335,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.043120701Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.043329962Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"81ada18c-c657-4cc1-9ac5-4dc5b987faed","remote_addr":"[fd00:1122:3344:101::c]:49205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.047977529Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36951"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.04809192Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.090283697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.150846352Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c77d744b-8551-444f-b613-fb73cf183475","remote_addr":"[fd00:1122:3344:101::b]:36951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102790,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.151090743Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.151227844Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f9d23242-16d7-42f4-a92e-8acd7481057f","remote_addr":"[fd00:1122:3344:101::c]:48191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":208,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.151387935Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.158765988Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a4b7e507-94e5-4fae-b9f2-c5e627ce3c4c","remote_addr":"[fd00:1122:3344:101::b]:36951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7464,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.170571537Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.17106593Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"467ee50d-b575-475d-90b2-e62e913a1cb2","remote_addr":"[fd00:1122:3344:101::a]:54690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":281,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.177480368Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.177695899Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"403b88b6-38c6-4ec7-99ae-78b08fc94910","remote_addr":"[fd00:1122:3344:101::a]:59904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":160,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.190864026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.195027321Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c377b76d-78fd-438a-9ac0-5a3c001897d0","remote_addr":"[fd00:1122:3344:101::a]:54690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":949,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.248077141Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"23f8c2d1-2e3b-48e0-a7d1-c704c5ecd53b","remote_addr":"[fd00:1122:3344:101::c]:37217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96864,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.248556424Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b54662b2-a348-4847-8793-65ec9a62933e","remote_addr":"[fd00:1122:3344:101::c]:37217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.250727817Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:32975"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.251483032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.286786508Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.342249853Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fe9938f6-0189-4b52-851f-c4fdbafd5a80","remote_addr":"[fd00:1122:3344:101::a]:32975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91455,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:24.342510285Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"595d4ac6-37cf-4bf2-8374-8a1dbdf9c74c","remote_addr":"[fd00:1122:3344:101::a]:32975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:28.253204827Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:28.253376558Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bc78e4fd-10bf-4e3b-b612-7231df309ea3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51264","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.002655564Z","hostname":"buskin","pid":1562,"uri":"/61e07fcc-45b4-4b25-ae0d-e78eee62eb60","method":"GET","req_id":"d3ab5b9e-e2f4-4070-a146-e1628287c5a2","remote_addr":"[fd00:1122:3344:101::d]:47540","local_addr":"[fd00:1122:3344:101::1]:42327","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"61e07fcc-45b4-4b25-ae0d-e78eee62eb60","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6775,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.110836628Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.111613673Z","hostname":"buskin","pid":1562,"uri":"/vmms/760054d9-c981-48b3-88ba-f2e26d33bb6c/state","method":"PUT","req_id":"aefea8bd-d4b0-4363-bf27-13daaa75d465","remote_addr":"[fd00:1122:3344:101::a]:39409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":715,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-09-09T17:59:32.111713153Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.111740724Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.111755904Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-09-09T17:59:32.111713153Z }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.111770564Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-09-09T17:59:32.111713153Z }, migration_in: None, migration_out: None }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-09-09T17:59:32.158078734Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.158104843Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.158133671Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-09-09T17:59:32.158078734Z }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.158141743Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:630","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.158278328Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.15847902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.158523758Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/6830ec9d-22cb-45c0-ba9d-5a3728bef44f/debug/bundle/zone\", \"/pool/int/d05e61ff-a727-4851-80ca-eab919525a00/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/e68e093e-0654-4278-a79f-9c234ab4cc35/crypt/debug\", \"/pool/ext/f5eda459-d1e9-44f5-b731-f6490e3fc719/crypt/debug\", \"/pool/ext/677df3fa-a1f5-4e63-93ee-592a272d5e3f/crypt/debug\", \"/pool/ext/89ffd24b-1dd8-4f52-85d7-6f3c66fce582/crypt/debug\", \"/pool/ext/46b15cce-a870-48ea-aab8-5f90f01b99e4/crypt/debug\", \"/pool/ext/0720cae8-ac21-4fb2-9b27-8f836e4c5c41/crypt/debug\", \"/pool/ext/f8e6d791-b5fc-4ef9-b2c8-37d1fd7dd299/crypt/debug\", \"/pool/ext/2c079939-1543-4f7b-ab38-d421cbf95e07/crypt/debug\", \"/pool/ext/b6fbc8d5-08ec-4cd5-ae5d-395890772649/crypt/debug\"] }","zone_name":"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:32.802409519Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1137","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c\", bundle_id: b4d820d9-b646-47f8-95e3-9e7aa8a2bb6c }, time_created: 2024-09-09T17:59:32.158740556Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_760054d9-c981-48b3-88ba-f2e26d33bb6c","v":0,"name":"SledAgent","level":40,"time":"2024-09-09T17:59:32.80250049Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:826"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.55391804Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.569828483Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(6460566), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.569879144Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(6460566), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.570888689Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-09-09T17:59:34.570863819Z }, migration_in: None, migration_out: None }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.600920285Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-09-09T17:59:34.570863819Z }, migration_in: None, migration_out: None }","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:34.605453502Z","hostname":"buskin","pid":1562,"instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","propolis_id":"760054d9-c981-48b3-88ba-f2e26d33bb6c","instance_id":"9a69ee61-cd07-4e16-8bce-71a799ee7a11","component":"InstanceManager","file":"sled-agent/src/instance.rs:763"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.121944258Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.122073409Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9524703b-75c3-421f-9a7d-55816ffd5f49","remote_addr":"[fd00:1122:3344:101::c]:54018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.13078032Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"40c3d96d-a1a7-42f5-877a-9bad820f7720","remote_addr":"[fd00:1122:3344:101::c]:54018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":119,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.25028642Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.250456451Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"734f852e-67fc-4283-bd9a-87b617107673","remote_addr":"[fd00:1122:3344:101::c]:41452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.619353082Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50099"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:35.619546283Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c922ee82-4ff7-48a7-9c34-2e4b2d2ceb98","remote_addr":"[fd00:1122:3344:101::a]:50099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:36.651363478Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:36.652171902Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"11e086ce-5052-45b8-9327-60d6b6fcb8e3","remote_addr":"[fd00:1122:3344:101::a]:61443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":766,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-09T17:59:36.66029269Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"742872e1-9d2e-4af1-9757-71b3d6367817","remote_addr":"[fd00:1122:3344:101::a]:61443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":146,"response_code":"204"}