[ Sep 4 16:48:25 Enabled. ] [ Sep 4 16:48:25 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Sep 4 16:48:25 Method "start" exited with status 0. ] [ Sep 4 16:48:25 Rereading configuration. ] [ Sep 4 16:48:25 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:25.907409651Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:25.907563634Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:25.960577067Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:25.960619448Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.083530916Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.092363337Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.101235498Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.700522356Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.840283466Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.840316347Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.84043877Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.840556033Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.840581023Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.840591584Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843655197Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:26.843664697Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843681707Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843688297Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843694747Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843743019Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843846771Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843878072Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.843893462Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.855194832Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.855226352Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:26.975566759Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466405343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466446484Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466452184Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466467085Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466472865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466491005Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.466495325Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.474985878Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.475009538Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:27.578556825Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066423318Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066458039Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.06649706Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.06651036Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066520981Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066526461Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066571072Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066578532Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066595442Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066600833Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066605673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066622783Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066628153Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066632223Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066650074Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066655134Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066659734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066664494Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066668624Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066678175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066683264Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066710905Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066717415Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066721505Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066726176Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066730036Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066749566Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066753716Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066758946Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066763217Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066768867Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066773417Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066800847Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066805438Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066810088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066817308Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066823908Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066829788Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066852519Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066867319Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066872129Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066877209Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066882029Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066886729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.06689258Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.06692001Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066939361Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066944791Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066948691Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066953471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066957471Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066961881Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066993542Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.066998782Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067003052Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067014612Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067024733Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067054613Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067059293Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067063244Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067067244Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067082944Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067098405Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067105545Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067117565Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067122745Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067130355Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067158746Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.067171706Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.078123997Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:730"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.078149738Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999910028s","next_cleanup":"Instant { tv_sec: 46092, tv_nsec: 375995454 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.07823964Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.07824772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.078322382Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.078341802Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081400795Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081466967Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081667761Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1053"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081673472Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081738183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081783274Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081797075Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081801785Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081825595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081848836Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081854066Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081858056Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081861676Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081865576Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081883147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.081915407Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082239535Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 54308cdc-0331-4daa-84cf-3ba0a405e364,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: d5abb5c3-38c8-4f7b-a191-64252eb83a69,\n rack_id: 54308cdc-0331-4daa-84cf-3ba0a405e364,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082530882Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082553423Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1187"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082611084Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082628364Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.082953862Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083076735Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083106386Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d5abb5c3-38c8-4f7b-a191-64252eb83a69, rack_id: 54308cdc-0331-4daa-84cf-3ba0a405e364, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083247659Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43761","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d5abb5c3-38c8-4f7b-a191-64252eb83a69, rack_id: 54308cdc-0331-4daa-84cf-3ba0a405e364, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083367022Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083379962Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083409883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083416583Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083420243Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083435374Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083440204Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083464314Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083474665Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083484065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083487725Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083492635Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083497965Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.083502715Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.085179485Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4163"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.085231566Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.116108672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.137654075Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.154941367Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.162869846Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:388"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.162927148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.162964929Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:400"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.162970809Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 143.261384ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.16638601Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f104f575-cf30-493a-83d9-e93e2d10b6a5\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f104f575-cf30-493a-83d9-e93e2d10b6a5\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_b11bf0e4-5e3b-4abd-af88-22ad6494addc/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.175877086Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_b11bf0e4-5e3b-4abd-af88-22ad6494addc/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.204849627Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_b11bf0e4-5e3b-4abd-af88-22ad6494addc/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.204893408Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.210197404Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.210206914Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:127"} {"msg":"Ensuring dataset oxi_b11bf0e4-5e3b-4abd-af88-22ad6494addc/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.210215284Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:137"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.30318833Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:172"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.310476873Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 169.393975ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.311072057Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"52bb700d-1298-47e0-ae9d-cebd336aecb5\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"52bb700d-1298-47e0-ae9d-cebd336aecb5\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.481940908Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 93.312675ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.482544483Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3779d193-aabd-4d31-ae97-0722c3e17344\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3779d193-aabd-4d31-ae97-0722c3e17344\" }"} {"msg":"Mounting oxi_b11bf0e4-5e3b-4abd-af88-22ad6494addc/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.522868113Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:180"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.536133889Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.536223342Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.536251632Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.536263993Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:194"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.576284056Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 138.43946ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.576798308Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6903b8a9-9371-475c-bf27-6a22cfa2c15c\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6903b8a9-9371-475c-bf27-6a22cfa2c15c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.716935587Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 237.385056ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.717601363Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3bc91f8e-a8c5-4bb9-a85b-e99706c66b16\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3bc91f8e-a8c5-4bb9-a85b-e99706c66b16\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.956451584Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 248.812423ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:28.957414287Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27c7f24c-3ede-4de6-b5b9-7d3ab1868f99\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27c7f24c-3ede-4de6-b5b9-7d3ab1868f99\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.20772135Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 309.364931ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.208176681Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f456b3ec-62db-4e06-9d95-6c350697666e\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:48:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f456b3ec-62db-4e06-9d95-6c350697666e\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.281783865Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.287595893Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.287627954Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.287702116Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:62744","producer_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:29.288967096Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"368.781831ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.289717164Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.289724284Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.289788845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:914","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.289794425Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:29.518921145Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:29.659237788Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"728.252614ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:30.389379273Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"705.601103ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:31.097181646Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.327213236s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601684436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2639"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601746078Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2857"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601756118Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2509"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601763618Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2556"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601769958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2938"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.601776328Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.603599302Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2782"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.604859672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2550"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.604950984Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:31.605094718Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 61.180285ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:31.817835436Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 110.336712ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:32.016759815Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 261.674912ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:32.19196844Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 362.438043ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:32.533018405Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 479.423507ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:33.100774602Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:33.426695777Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.534596374s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.099256617s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:33.654763791Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.319285238s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:34.83203623Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.288272155s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:36.252121713Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917279566Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917330417Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917339327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917358588Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917629164Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917820808Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.91788426Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43761","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.91789895Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:37.917910661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180576678Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.18065769Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180787133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180875955Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180882956Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180886676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180921656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.180941807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:38.181528741Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.124856607s"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.185935626Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4187","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.185962487Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4203"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.185967377Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.437471639Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63876"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.437791637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.43795202Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"afb02d5e-1d23-46e2-9c11-d326dceda7c3","remote_addr":"[fd00:1122:3344:101::1]:63876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":347,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.468262753Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50209"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.468342535Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"0fc36bfa-24e9-49dc-b6a8-f70fe3b26c43","remote_addr":"[fd00:1122:3344:101::1]:50209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.46898696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.469017561Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.46940596Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.469727877Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:1001"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: bdd39113-8955-4f55-9aeb-37b20c3ca0cc, pool_id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: fb8ddddf-522c-49b6-97b6-6a876aea96dc, pool_id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1c4e0669-0784-4947-a4bb-5553b4c8fe64, pool_id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b9694e07-1532-4693-aa70-351df51c090f, pool_id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: b7f2c7b9-4e5f-4ee8-970f-f039ab972e4f, pool_id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 71215cc5-5bd3-440b-9964-5fa592f5d841, pool_id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: f8356e07-0de9-4eed-9501-d6d300ff369b, pool_id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: c68d3b2f-91cb-4d89-8e9f-ef06d736e5c9, pool_id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4f889948-0f68-43b2-804d-9465efd7bc8c, pool_id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502260163Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502422307Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40284"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502687733Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:856"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: bdd39113-8955-4f55-9aeb-37b20c3ca0cc, pool_id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: fb8ddddf-522c-49b6-97b6-6a876aea96dc, pool_id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1c4e0669-0784-4947-a4bb-5553b4c8fe64, pool_id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b9694e07-1532-4693-aa70-351df51c090f, pool_id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: b7f2c7b9-4e5f-4ee8-970f-f039ab972e4f, pool_id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 71215cc5-5bd3-440b-9964-5fa592f5d841, pool_id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: f8356e07-0de9-4eed-9501-d6d300ff369b, pool_id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: c68d3b2f-91cb-4d89-8e9f-ef06d736e5c9, pool_id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4f889948-0f68-43b2-804d-9465efd7bc8c, pool_id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502724894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502732114Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502739074Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502745164Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502753175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502759315Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502766475Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.502772155Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.649440909Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:38.64946774Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.099207845Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:39.101035478Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.167552323Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.17497179Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175042991Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175052272Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175065552Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175073512Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175177744Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.175184175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.215764962Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_62c25a09-dd5a-42bc-bdc0-88462c2034ea/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.2232375Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_62c25a09-dd5a-42bc-bdc0-88462c2034ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.432890885Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.440907726Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.440974497Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.440983388Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.440991118Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.440997568Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.450699799Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.45073264Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.562064072Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.569714095Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.569749985Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.569757836Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.569787116Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.569798467Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.5699201Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.56992739Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.613270502Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9859c4c0-8dca-4c62-8980-22f58b7298bf/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.62158152Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_9859c4c0-8dca-4c62-8980-22f58b7298bf","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.826115333Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.834006392Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.834040282Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.834047522Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.834051873Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.834056723Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.843281402Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.843305983Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.961028478Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.969065489Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.96910847Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.96911439Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.96912795Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.96914022Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.969274734Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:39.969281754Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.01148938Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e5004d09-cef9-4e6f-b2aa-6d7ade3877f4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.019454009Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:40.103800429Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_e5004d09-cef9-4e6f-b2aa-6d7ade3877f4","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.230246981Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.238534779Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.23858784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.238615141Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.238623911Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.238630471Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.248354383Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.248383954Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.36279785Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371031836Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371067857Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371079027Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371092957Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371105238Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371258401Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.371268261Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.41402973Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f73a9d5e-d846-41dd-90e4-dedfd04db60a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.422236216Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_f73a9d5e-d846-41dd-90e4-dedfd04db60a","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.629956005Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.637815612Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.637844343Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.637851493Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.637857053Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.637868313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.648464166Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.648501717Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.763795584Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771700542Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771741303Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771756583Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771786784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771793854Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771931518Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.771943958Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.816967611Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_84deb833-6610-41c6-8b5e-44fa2aeee752/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:40.826338074Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_84deb833-6610-41c6-8b5e-44fa2aeee752","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.032619988Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.040461435Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.040510946Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.040555187Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.040564027Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.040568507Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.050420572Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.050449893Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.162789249Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.170852931Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.170896432Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.170906873Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.170944304Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.170958704Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.171121258Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.171129198Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.211718945Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:41.215728871Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Automatically destroying dataset: oxp_41da29c9-1a43-49cc-b472-5dde9cb41a62/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.220241898Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_41da29c9-1a43-49cc-b472-5dde9cb41a62","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.430531709Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.439099553Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.439134563Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.439142964Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.439153304Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.439160474Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.44864336Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.448670841Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.570561055Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.578101674Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.578138635Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.578146555Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.578184556Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.578198167Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.57833543Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.57834661Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.620684569Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.628423183Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_d2efc072-fbdb-4b59-a671-9eb348ded4ad","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.830686823Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.838700903Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.838740044Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.838746735Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.838754725Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.838759765Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.848506017Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.848541268Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"5f5397b7-7011-4692-88c2-8a70902c2fea (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.970291238Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.977023418Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.97707642Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.97709191Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.97710493Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.977123511Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.977246134Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:41.977256884Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.021810996Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5f5397b7-7011-4692-88c2-8a70902c2fea/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.030730338Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:42.218558363Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_5f5397b7-7011-4692-88c2-8a70902c2fea","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.240794883Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.248946967Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.248990428Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.249003338Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.249012309Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.249020629Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.258883304Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.258907105Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.374395376Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382233143Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382266793Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382275484Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382289974Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382304554Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382428117Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.382438358Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.425707039Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a145c7dc-0acd-4e57-96ae-74d5f1432797/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.43332806Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_a145c7dc-0acd-4e57-96ae-74d5f1432797","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.635771453Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643551949Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.64359577Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643669912Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643828785Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b11bf0e4-5e3b-4abd-af88-22ad6494addc (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 46c20aa9-c404-447d-81be-a48fc73c6ec0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643912278Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643922428Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.643985769Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644197844Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644208674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644212855Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644217245Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:891"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644228355Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:896"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644265946Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644274826Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:902"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644313097Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644320087Z","hostname":"buskin","pid":1562,"sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.644404729Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"e2204290-be60-4a9b-9fb2-5f0a7871532a","remote_addr":"[fd00:1122:3344:101::1]:40284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4141864,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.67635356Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.676468793Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39568"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.676705459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.676715039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.736467163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.736499213Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.795654163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.795691734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.853613414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.853647295Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.853665845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.853703606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.896496225Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.896535356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.896543997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.945838701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.949470367Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.953103244Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.965690774Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.970002916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:42.973791946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:43.22115714Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:43.307748003Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.570235982s"} {"msg":"Profile for oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:44.517420174Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:44.517514666Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:44.517612528Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.00080459Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.481278338Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.481345039Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 46.594078ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.488316265Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.488367996Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.488440488Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:45.492787562Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 89.761988ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.545988769Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 32.445261ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.618607539Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 44.471945ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.62494661Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 244.9839ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.643975354Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 145.147325ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.657838344Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 90.245671ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.676292394Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 259.38371ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.878279166Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 258.020518ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.905536266Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 418.692052ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:45.964589443Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 400.38868ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.107178839Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 246.233342ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.175983029Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 899.683854ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.391410751Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 531.084538ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.430268207Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 554.555364ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.515110789Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 578.751306ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:46.971163944Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 914.347833ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:47.082998598Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 655.428156ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:47.301900554Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 1.435602048s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:47.559995623Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:47.903053257Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4211"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:47.903107138Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:47.908815324Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4300"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 1.23904064s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:47.965998926Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 1.307836509s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:48.006958852Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.048597734Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.048639555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4375"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.091698291Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4388"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.091733741Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4252"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.14537466Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4294"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.14541228Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4355","rack_subnet":"fd00:1122:3344::"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.148201057Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:52554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.149271032Z","hostname":"buskin","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"979199c4-1eb2-4e88-8271-3a68db37c9db","remote_addr":"[fd00:1122:3344:101::2]:52554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":976,"response_code":"200"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.166992205Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4366"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.167031315Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4408"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.254863998Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4440"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.254899629Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.362267457Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4487"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.362326649Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:48.396647236Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.396964044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.429411017Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.42951878Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.430029532Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.430065402Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.466808018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514646308Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514710349Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.51472373Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.51473834Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514763611Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514769551Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514776861Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514782731Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514788471Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514795471Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514799781Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514806972Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514812722Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514818122Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514825182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514831002Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514841062Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514845302Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514849162Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514853093Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514857343Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514861283Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:48.514865233Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 745.936294ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:49.004075158Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9\") failed: Property not found. retry in 1.436602208s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:49.212945205Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 1.035675207s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:49.323071928Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 830.115273ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:49.760935221Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56\") failed: Property not found. retry in 1.326267032s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:50.36743764Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012\") failed: Property not found. retry in 1.452419054s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:50.600607496Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:51.882253131Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"no record found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"38.818579913s"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.07278508Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.072826851Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_5e402c3a-7e0c-49f4-9e03-e03a9a42f012"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.072835661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_a389897c-961c-454c-9a47-71a26222a8f9"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.072882952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.07321291Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.073541558Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"08939447-bc49-4ec8-94ae-219ea8c49bd3","remote_addr":"[fd00:1122:3344:101::1]:39568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9396994,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.075538486Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.134330886Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.186563691Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.241211923Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.241236283Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.273600445Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.275947461Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.275953461Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.305925325Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.306097369Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61649"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.306297733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.306325684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.326992477Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.327023447Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.345650841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.345689962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.362587815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.362680347Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.362902332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.363015585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.374439797Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.407550096Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.408180761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.408206181Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.437805837Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 227, 126]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.437823437Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.438113404Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.438121804Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.438418501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.439586099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.46312903Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:52.480707549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:53.120033541Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:53.120110222Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 33.400226ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:53.2513591Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 98.90103ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:53.291931516Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 168.205644ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:53.429044963Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 323.669932ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:53.608234262Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 1.153062506s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:53.940021457Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 754.873533ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:55.104103531Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 1.041662531s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:55.867334546Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 1.144921841s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:56.917886285Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\") failed: Property not found. retry in 906.600356ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:58.071229963Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:58.997479121Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:58.997555923Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:58.997898541Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:58.998260029Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b6884249-ffa2-434c-86a5-be8e400d50e8","remote_addr":"[fd00:1122:3344:101::1]:61649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6692105,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.000549544Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.030147929Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.030300523Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40784"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:48:59.038278683Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.038425766Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"65c37b01-17d7-4b65-b927-1fd21b7a74b9","remote_addr":"[fd00:1122:3344:101::1]:40784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8014,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:48:59.041634173Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.134369832Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:48:59.134481795Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53259"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_e5dbb7cb-9b3b-4016-a67c-456d1beabcd1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:49:03.142637849Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.142769592Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"bc144ea2-9a09-426e-a321-fd904b3f3289","remote_addr":"[fd00:1122:3344:101::1]:53259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008153,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:03.14603621Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.256110453Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.256229456Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.264998624Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"04142022-1313-4011-8a32-c296f8ad02a3","remote_addr":"[fd00:1122:3344:101::1]:36939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8674,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.267963665Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:03.267970475Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.508512117Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.508622109Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57263"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.516967668Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d6f509e6-f60d-4712-bd88-0e18901c29c4","remote_addr":"[fd00:1122:3344:101::1]:57263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8270,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.51830114Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:03.51830821Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.652906217Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.65302402Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42893"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.661511852Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"374c4959-8e23-487d-b91d-ef2401973a14","remote_addr":"[fd00:1122:3344:101::1]:42893","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8386,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.664581655Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:03.664593835Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.81843671Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.818539353Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.826873312Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"14191a1a-748f-4d9b-b4d6-6ca439b6b142","remote_addr":"[fd00:1122:3344:101::1]:52751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8267,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:03.828252104Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:03.828274735Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.145712798Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.14580427Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.153564135Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"a2260044-d868-4f27-9a6c-94bf997522bd","remote_addr":"[fd00:1122:3344:101::1]:56115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7705,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.154723182Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:04.154735923Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.528090068Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.52818536Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.535239748Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"5826d39b-2fa5-42a4-a2df-30783da19d5c","remote_addr":"[fd00:1122:3344:101::1]:57758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6933,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.537044431Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:04.537051942Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.784066877Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.784155989Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.791158646Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"912f30f4-0099-48c9-aa83-9ae7ca7397ed","remote_addr":"[fd00:1122:3344:101::1]:48340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6911,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:04.792873927Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:04.793361058Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.327921974Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.328048847Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:58251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.335465293Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"87956dd5-0dd0-4ecf-9798-68282ec2ed1f","remote_addr":"[fd00:1122:3344:101::1]:58251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7316,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.33867587Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.33868533Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.146018418Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.146130181Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.166966527Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"0d2c7706-ffbb-4570-8e3c-fba17a56e173","remote_addr":"[fd00:1122:3344:101::1]:51339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":20660,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 94484628, ip_addr: 5.161.184.148, stratum: 3, ref_time: 1725468544.8331385, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.168714079Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.168720879Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1266"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.198402816Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.198538569Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56585"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.199511462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.199518742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.220374339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.22041637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.238258996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.238299626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.255628359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.25566517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.317397161Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.317437222Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.377526214Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.377571355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.440072194Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.440110295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.503044184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.503095925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.564099789Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.56415154Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.564401146Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.564540519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595449466Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595494967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595503377Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595511247Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595519717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.59562912Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"91f07fd7-95c8-4975-8910-892def11051d","remote_addr":"[fd00:1122:3344:101::1]:56585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\", TimeNotSynchronized), (\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\", TimeNotSynchronized), (\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\", TimeNotSynchronized), (\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\", TimeNotSynchronized), (\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\", TimeNotSynchronized)]","latency_us":396880,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.595812084Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"50.155436ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91f07fd7-95c8-4975-8910-892def11051d\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91f07fd7-95c8-4975-8910-892def11051d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.647953717Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.648292565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.648306765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.666302384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.666341675Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.684320463Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.684371134Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.701924753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.701967954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.719737237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.719783888Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.738010842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.738048353Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.755164491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.755218922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.773405206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.773445097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.789789866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.789847147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.790040982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.790186055Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.821630295Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.821690836Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.821701166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.821710786Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.821718747Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.82184985Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"bc186d08-3b3d-4270-a95c-aab3b94dbf7d","remote_addr":"[fd00:1122:3344:101::1]:56585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\", TimeNotSynchronized), (\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\", TimeNotSynchronized), (\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\", TimeNotSynchronized), (\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\", TimeNotSynchronized), (\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\", TimeNotSynchronized)]","latency_us":173730,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:05.822000833Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"65.812352ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc186d08-3b3d-4270-a95c-aab3b94dbf7d\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc186d08-3b3d-4270-a95c-aab3b94dbf7d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.888142629Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.888420616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.888436096Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.907576222Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.907616483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.925408547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.925449708Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.942146906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.942189987Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.959476239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.95953346Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.978881151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.978919792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.998315794Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:05.998360055Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.007683503Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.007731065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.026018111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.026072372Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.026260096Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.026381829Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.108322651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.108384502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.108393983Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.108401163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.108409973Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.195725123Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.199957464Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.204338619Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.208767784Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.213003195Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.217511632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.221904007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.226349713Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.230910652Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:06.235295106Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.075208161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.075354274Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.084554904Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.084699238Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.090480296Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.090613299Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.09820117Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.098329194Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.185128837Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:10.18525301Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 45.12578ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.226206729Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 59.866551ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.234893866Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 72.567106ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.242336374Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 25.95ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.249116716Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 87.702955ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.278715283Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 102.198429ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.290908794Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 112.097381ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.497987151Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 246.56643ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.673771481Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 132.549012ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:10.845447122Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 132.664686ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.053616725Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 262.899176ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.113025114Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 431.832241ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.255482378Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 498.561022ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.314983489Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 60.718426ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.383355282Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 100.511356ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.557004391Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 492.317515ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.649198874Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 147.876878ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.687787815Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 363.855065ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.766625839Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 1.015196988s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.856491566Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.180863968s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.890818596Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 246.333849ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:11.915336362Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 664.545165ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.148130063Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 494.389952ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.165475057Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 225.265032ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.187399131Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 878.839095ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.436165854Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.088731086s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.681540766Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 1.427401557s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.838013924Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 908.777087ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:12.889621477Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 759.6002ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:13.089201594Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 1.185284037s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:13.325240304Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 786.602904ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:13.777337754Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 586.015146ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:13.80436184Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.492346929s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:13.854783415Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 1.256306761s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:14.270602718Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 812.887101ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:14.397983061Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 960.434307ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:14.516879122Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.380653621s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:14.571194389Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 1.406278421s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:15.219646582Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.196316173s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:15.353205184Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 1.437313623s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:15.484038861Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 1.447831668s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:15.532946029Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 909.810324ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:15.957252569Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.160529734s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:16.556509119Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 628.364681ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:16.631981333Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 945.920363ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:16.874389445Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 1.138380265s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:16.926642644Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 647.194019ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:16.985268955Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 1.048003796s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:17.270867819Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 562.416087ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:17.640123963Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.030694442s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:17.723672239Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.412251052s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:17.826949007Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 950.748881ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:18.072377692Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:18.107701336Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:18.108607377Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"38f4308f-60d8-4746-8eb6-ad20b8100447","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62264","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":811,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 1.16379488s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:18.210693426Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 1.442884752s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:18.324223719Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.005023449s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:18.760669538Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 673.002924ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.029529741Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.124420995s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.243517864Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 597.144054ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.38066512Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac\") failed: Property not found. retry in 1.387030216s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.710781438Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 949.098307ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.771914658Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 532.497281ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.779732945Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 857.294151ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:19.985549162Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 929.037175ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:20.322528853Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.055898003s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:20.373392618Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de\") failed: Property not found. retry in 1.227379301s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:20.727417035Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 578.005222ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:20.848902798Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979\") failed: Property not found. retry in 1.290480167s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:21.257688863Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04\") failed: Property not found. retry in 595.016038ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:21.435611924Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c\") failed: Property not found. retry in 1.38196129s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:21.44465196Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.84448469Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_4dec1158-5c1a-4225-8e1d-6b8e2f11748c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.844553432Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_0a419753-947c-4e72-9631-ad91417fc1de"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.844563562Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_b5222f50-3cda-4d9b-9e13-07be3e4b1979"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.844569872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_eeb8af3a-e1a6-41fc-aa51-1a104791b6ac"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.844576552Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_ee268c38-8f2f-4e2c-bc1f-b470afd9fa04"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.844611553Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.845004922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.845366841Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5a483308-92f5-45f0-ba73-220f14f247bb","remote_addr":"[fd00:1122:3344:101::1]:56585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":16087731,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.879787573Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:35092"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:22.906917622Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3686"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:31.934603926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3704"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.227331148Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3727"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.228086376Z","hostname":"buskin","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"10844119-6b05-438d-a6ca-917c7031423a","remote_addr":"[fd00:1122:3344:101::1]:35092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":13384317,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.368023659Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.368435549Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38800"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.36975646Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.369768121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.394093172Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.394123442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.414891089Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.41492953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.434477077Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.434514247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.457684511Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.457715912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.478049578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.478082088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.499625663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.499658524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.518595616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.518637547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.538305727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.538336548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.62425289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.624289871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.715301655Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.715341776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.795064521Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.795104252Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.872310037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.872346977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.951380305Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:36.951416966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.030420803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.030458974Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.112113705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.112150976Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.186492082Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.186541513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.251580407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.251613687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.321171129Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.32120709Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.395411733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.395449044Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.4660181Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.466062781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.466318657Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.466616674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577255857Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577315659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577324339Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577330039Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577335779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577342979Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577349189Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577355049Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57736242Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57736835Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57737564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57737957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57738482Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57739025Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.57739421Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577399501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577417591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577423481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.577428901Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.577557524Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"255f0c0a-c957-4ff4-a474-9a35a7d65b06","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized)]","latency_us":1212105,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:37.578298212Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"104.523763ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"255f0c0a-c957-4ff4-a474-9a35a7d65b06\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"255f0c0a-c957-4ff4-a474-9a35a7d65b06\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.683855804Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.684455428Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.684467188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.706474414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.706502745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.728963562Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.728988512Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.750081796Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.750108372Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.770704229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.770728719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.789876417Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.789904977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.809418493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.809443964Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.831162093Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.831189424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.85197863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.852017541Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.872339477Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.872364007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.893430181Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.893454181Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.913256164Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.913283084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.936098049Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.93612634Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.957021799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.95704716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.977171971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.977197011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.996284027Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:37.996317248Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.018337554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.018369075Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.041695882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.041725783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.061584427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.061615198Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.082946067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.082971578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.104006141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.104034881Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.104264327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.10439716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202618226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202673078Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202679958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202685548Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202702188Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202707999Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202715069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202720699Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202724849Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202730579Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202737429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202743209Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.20274732Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.202751549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.20275719Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.20276276Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.2027682Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.20277384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.20277925Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.202895947Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a0f7133a-29b6-4308-a58e-15f86f2133bf","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized)]","latency_us":520060,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.203054447Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"148.286698ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0f7133a-29b6-4308-a58e-15f86f2133bf\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0f7133a-29b6-4308-a58e-15f86f2133bf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.352136038Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.352476108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.352491286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.375185029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.37522322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.394276175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.394310206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.415104293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.415136183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.435469979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.43550101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.456116993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.456146993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.476160148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.476195092Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.495120601Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.495150455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.514701652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.514735793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.535922309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.535970389Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.555842535Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.555876506Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.575916104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.575945205Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.595643486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.595673887Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.616357041Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.616387831Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.63598597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.636026571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.655019584Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.655048355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.675868123Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.675897673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.695375018Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.695404959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.714808093Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.714839594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.734389361Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.734422811Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.754699026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.754742607Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.754981842Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.755114156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873307759Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873362021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873374041Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873381051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873388491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873394221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873400351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873407162Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873421172Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873425422Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873430512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873436032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873441352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873447863Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873453523Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873459043Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873464323Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873469933Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873475173Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:38.873611016Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0c095d7f-a4ab-459b-9e5d-7a0f6ddc7529","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized)]","latency_us":522747,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:38.873811051Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"196.699391ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c095d7f-a4ab-459b-9e5d-7a0f6ddc7529\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c095d7f-a4ab-459b-9e5d-7a0f6ddc7529\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.071593496Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.071948565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.071956685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.096829029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.09689099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.120084444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.120124395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.141159458Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.141200569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.162416386Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.162458037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.183190632Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.183226932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.20484954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.204898581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.225035572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.225088543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.244463546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.244507607Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.266124283Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.266161974Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.286378477Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.286418368Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.30657103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.30661308Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.327800667Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.327841097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.347150909Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.3471878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.366608704Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.366645045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.38736904Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.387416041Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.407416239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.4074599Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.428480802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.428529323Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.44933681Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.449373061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.469357599Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.46939902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.489375887Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.489442019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.489687534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.489836578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591786884Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591832685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591841485Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591847705Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591853255Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591860445Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591864875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591870735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591876415Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591882106Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591885786Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591892926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591907216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591914487Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591920147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591925267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591932157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591937817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.591943047Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.592087641Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3959528d-df2e-4832-950e-a8f826e2d2ca","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized)]","latency_us":521722,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:39.592246624Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"127.591515ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3959528d-df2e-4832-950e-a8f826e2d2ca\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3959528d-df2e-4832-950e-a8f826e2d2ca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.720329594Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.720645772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.720653772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.744229225Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.744268456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.765661336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.765712759Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.787064069Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.7871041Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.807384904Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.807420145Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.829929363Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.829964803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.851445355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.851480408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.873842992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.873891953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.89638388Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.896424621Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.918073678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.918109229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.940907353Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.940950874Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.962124539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.962163811Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.983626894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:39.983669615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.005402624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.005446235Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.026710723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.026752854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.047558631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.047602792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.067882517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.067918318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.087387473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.087432674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.106730517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.106767476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.126517667Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.126552188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.146623562Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.146671609Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.146887004Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.147016727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243470622Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243522393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243528403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243531963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243536313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243540643Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243545603Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243550003Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243554764Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243558894Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243563834Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243567484Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243572494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243576724Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243581034Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243585084Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243594615Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243599215Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.243603545Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.243693687Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c0ffc0ad-dc71-4eed-8c73-995a21977a59","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized)]","latency_us":524650,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.24384027Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"239.516546ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0ffc0ad-dc71-4eed-8c73-995a21977a59\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c0ffc0ad-dc71-4eed-8c73-995a21977a59\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.484618083Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.48491369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.48491979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.509152949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.50918738Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.528972362Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.529003383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.548819656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.548850887Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.569822058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.569857889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.589260943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.589292793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.609503026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.609532527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.630204001Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.630237072Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.648721703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.648756294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.667937482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.667963393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.68626718Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.686294471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.708051211Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.708079221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.728739085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.728769206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.747861562Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.747903032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.76665212Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.766682281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.787844427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.787873046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.808014869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.808045149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.828645501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.828815106Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.849384817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.849413618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.868615856Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.868641727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.887238721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.887272472Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.887500258Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.887612931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982494067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982538997Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982545898Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982549918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982555298Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982580118Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982585849Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982592719Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982598179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982603299Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982608329Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982613489Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982618649Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982625499Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.98262942Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.9826348Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.98263888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.98264246Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.9826473Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:40.982729022Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"254d75d2-295e-49af-b2ac-72b6bb91008d","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized)]","latency_us":499342,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:40.982880226Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"241.239304ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"254d75d2-295e-49af-b2ac-72b6bb91008d\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"254d75d2-295e-49af-b2ac-72b6bb91008d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.224062887Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.225029781Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.225045041Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.249586537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.249633367Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.274312298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.274358079Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.294659264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.294707785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.318476103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.318523684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.341192616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.341236587Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.362132556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.362183527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.38527563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.38533153Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.408986965Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.409037606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.433586413Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.433646674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.456340617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.456389038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.47949117Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.479532421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.502237123Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.502277594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.524901322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.524948126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.547696269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.547757351Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.572692596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.572739708Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.596009613Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.596064135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.617675201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.617720312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.63941697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.639466862Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.659590432Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.659630403Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.682738945Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.682799267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.683007492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.683155505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790826088Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790893529Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790900809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790908089Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.79091359Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.79091992Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.79092624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.79093218Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790954431Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790959801Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790964971Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790970321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790975491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790982381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790987691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790993142Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.790998502Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.791005452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.791010772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:41.791125075Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"49fc2d89-ea54-4387-baa2-57f3af7606c3","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized)]","latency_us":568515,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:41.791285679Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"398.05327ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49fc2d89-ea54-4387-baa2-57f3af7606c3\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49fc2d89-ea54-4387-baa2-57f3af7606c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.189750457Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.190075125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.190084135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.215284097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.215339468Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.238374979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.23842455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.259559105Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.259604026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.283052276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.283091637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.305051292Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.305092563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.327514258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.327565749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.348703965Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.348753386Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.370735431Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.370779952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.394101339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.39414298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.417298253Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.417334404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.439388561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.439422959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.46150742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.46154281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.483522566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.483567556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.505717766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.505756787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.526887502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.526938403Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.550364532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.550408703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.573010323Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.573050593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.596339131Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.596380062Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.618235404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.618272975Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.64194214Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.641989221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.642205686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.642323249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.746955629Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.74700861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.74701659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747022181Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747027731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747033061Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747038761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747044121Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747049421Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747053051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747059681Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747064922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747086362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747091762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747095992Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747100752Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747106172Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747109612Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747114663Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:42.747192825Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"61aa84df-1b99-449a-8ef1-228225a4a98a","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized)]","latency_us":558821,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:42.747373779Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"349.005933ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61aa84df-1b99-449a-8ef1-228225a4a98a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61aa84df-1b99-449a-8ef1-228225a4a98a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.096795846Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.097237817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.097246087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.119692543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.119740314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.137871427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.137913308Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.155642792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.155685853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.176738726Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.176775137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.197194155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.197240196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.216396223Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.216440355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.239893905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.239949126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.258913889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.25895321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.278432256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.278468597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.298519716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.298555246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.319122548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.319160289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.337386244Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.337428435Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.357760441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.357799032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.378974178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.379017028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.401265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.401303011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.422243231Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.422286992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.442958056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.443002207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.462421721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.462459532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.481584037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.48162694Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.503882952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.503932343Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.504150758Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.504277561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609475664Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609540026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609548786Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609554576Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609560236Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609567607Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609572107Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609577517Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609584487Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609590237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609594097Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609599777Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609603647Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609610828Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609617058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609622778Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609643278Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609649028Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609654158Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:43.609778232Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3be2e084-06a5-4ee6-89c9-65ad6f6a2a43","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized)]","latency_us":514186,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:43.609935405Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"516.774096ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3be2e084-06a5-4ee6-89c9-65ad6f6a2a43\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3be2e084-06a5-4ee6-89c9-65ad6f6a2a43\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.126968887Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.127321065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.127334315Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.152938537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.153003269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.175639569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.17568476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.197557393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.197600704Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.218697128Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.218744419Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.239486345Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.239532264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.25895177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.258996571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.278832125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.278888056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.300460311Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.300516773Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.322084408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.322129659Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.341665236Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.341705547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.362300489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.36234366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.385082073Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.385129928Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.408073871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.408122953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.429023077Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.429067034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.448837226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.448882117Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.468490286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.468549857Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.487050209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.48710054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.505327086Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.505370577Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.525453776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.525496897Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.546277944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.546330165Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.54652527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.546655463Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647476902Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647532413Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647537653Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647544093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647548213Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647562024Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647567354Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647572714Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647578994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647588454Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647594174Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647599625Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647604575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647609965Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647615255Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647622285Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647630945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647637406Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647642696Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:44.647778559Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ac0a1458-4f4a-493c-9047-619286bec2e9","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized)]","latency_us":522086,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:44.647981354Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"766.469225ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac0a1458-4f4a-493c-9047-619286bec2e9\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac0a1458-4f4a-493c-9047-619286bec2e9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.413787668Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.414133846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.414147087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.437853983Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.437903434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.462171504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.462222127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.485050821Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.485105072Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.505988081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.506031922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.528790546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.528836527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.548886356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.548930277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.569398976Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.569449797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.590256504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.590299565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.612954176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.613004628Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.633856566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.633897377Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.655955924Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.655998705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.676572006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.676619777Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.696280107Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.696328548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.715166618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.71521769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.735216217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.735258208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.757390467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.75743512Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.776729909Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.77677115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.798452398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.798494489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.819848819Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.81989415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.841737862Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.841792123Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.842008919Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.842127391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943251677Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943298938Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943304518Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943308758Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943312728Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943316808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943321598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943325588Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943338769Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943342139Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943346059Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943350289Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943354319Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943358509Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943362449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943366309Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.94337026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.9433735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.94337675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:45.943442871Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a3511ddf-7152-4f40-82ce-a720b8d15fb9","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized)]","latency_us":530950,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:45.943592925Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"396.267789ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a3511ddf-7152-4f40-82ce-a720b8d15fb9\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a3511ddf-7152-4f40-82ce-a720b8d15fb9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.340692021Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.341082301Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.341091011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.365538065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.365586786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.386207539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.38624294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.407423786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.407458677Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.428982461Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.429016782Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.450101125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.450137966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.469413057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.469454178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.490210273Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.490245445Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.510352775Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.510386519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.531408798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.531451389Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.552962233Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.552994404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.573862382Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.573894813Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.596007081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.596042522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.617380282Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.617414663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.637776849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.63781008Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.65705107Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.657097821Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.677103369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.677137109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.697775572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.697809133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.719329137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.719363028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.740535394Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.740574475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.761666149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.761717605Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.761918895Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.762038498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.859951087Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.859992248Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.859996628Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.859999798Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860004048Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860008148Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860012988Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860017028Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860021239Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860025288Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860030049Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860034179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860046449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860050879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860055009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860059949Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860063Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860067189Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.86007037Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:46.860152742Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"51005539-e44f-4cf7-bdc5-5d9dde6d6c6c","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized)]","latency_us":520716,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:46.860304235Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.108034294s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51005539-e44f-4cf7-bdc5-5d9dde6d6c6c\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51005539-e44f-4cf7-bdc5-5d9dde6d6c6c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:47.965890226Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:47.966205264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:47.966210884Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:47.990042454Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:47.990073384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.01292311Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.012953101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.035170312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.035201742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.055514348Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.055546048Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.064616325Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.068559409Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c9363003-1086-4843-8412-dcc236518232","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40914","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3829,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.076268763Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.076288214Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.095668937Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.095712348Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.115720626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.115754467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.136906652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.136935783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.157972525Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.158004136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.17868707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.178718101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.200794267Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.200822374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.226101043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.226132734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.249545793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.249576434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.271917517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.271948158Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.292072949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.2921084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.312475317Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.312504376Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.332810142Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.332840693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.352200226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.352227226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.37122006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.37124845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.390454269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.39049745Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.390696825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.390822598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492069327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492118048Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492124818Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492130669Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492137229Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492142719Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492148179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492154799Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492158539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492164979Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492170419Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49217581Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49218121Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49219515Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49220066Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49220429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.49220923Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.4922128Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492217891Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:48.492286672Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"908d6a46-1497-44ae-9d80-f3fdb53483b2","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized)]","latency_us":527692,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:48.492449686Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"610.197215ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"908d6a46-1497-44ae-9d80-f3fdb53483b2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"908d6a46-1497-44ae-9d80-f3fdb53483b2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.102192332Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.10251278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.10252264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.126753229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.12680033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.147660538Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.14770828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.168583978Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.16862853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.18833058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.188372281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.208236696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.208276177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.22765248Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.227696671Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.247149485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.247196936Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.266946498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.266990989Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.285385719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.28543407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.305021728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.305065409Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.323844218Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.323883708Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.343406785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.343464206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.362155033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.362193774Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.380977532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.381021833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.399217758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.399257399Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.418903418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.41894604Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.437678227Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.437716358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.458347281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.458383582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.477792696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.477826286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.497599799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.49764234Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.497859415Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.497989078Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590815916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590861347Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590887697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590893918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590900438Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590906178Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590911868Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590917478Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590922378Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590927088Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590933548Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590939479Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590945229Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590951069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590956839Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590961859Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590966559Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590977429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.590982449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:49.59100961Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"214ec9b6-7d87-4620-8743-7419a6e8a104","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized)]","latency_us":490029,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:49.591169794Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.095745418s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"214ec9b6-7d87-4620-8743-7419a6e8a104\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"214ec9b6-7d87-4620-8743-7419a6e8a104\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.685034656Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.685379314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.685393265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.70862356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.708662601Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.730235286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.730272847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.750696915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.750733056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.771208475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.771247326Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.791355076Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.791395377Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.811930578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.811968509Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.831556697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.831591767Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.851283418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.851320359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.869915032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.869945663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.889310916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.889343476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.908411932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.908448843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.929832524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.929865254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.951314497Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.951359992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.97194979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.971990531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.995968834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:50.996005725Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.022500678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.022536708Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.044554794Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.044583175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.065491685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.065520415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.085726658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.085752569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.106978166Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.107008206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.107272373Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.107395286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207805374Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207853396Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207859646Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207863816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207869196Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207886427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207892167Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207897607Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207902677Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207907917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207913287Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207918117Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207923177Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207928197Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207932918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207938018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207943038Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207948178Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.207952998Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:51.208084161Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5ae207cc-7bd0-4d52-962f-f39d1f6c38a2","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized)]","latency_us":524331,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.208254255Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.340941174s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ae207cc-7bd0-4d52-962f-f39d1f6c38a2\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ae207cc-7bd0-4d52-962f-f39d1f6c38a2\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:51.333136029Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"338.13605ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.546718151Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.547016853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.54702515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.569475244Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.569514516Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.590709192Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.590745563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.613788853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.613831993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.633689349Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.63374249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.655806157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.655842508Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.676141393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.676188354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.69654921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.696584581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.71664256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.716677981Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.736274719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.73631784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.757193029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.75723071Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.777136045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.777171966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.799171302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.799212903Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.821247299Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.821292104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.844213185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.844250849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.86482947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.864863611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.885579636Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.885619116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.905605564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.905639655Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.926849612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.926890293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.948024207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.948060038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.968613589Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.96865327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.968905905Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:52.96905056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.07244714Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072500111Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072507881Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072515252Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072521042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072527512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072532982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072556633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072563393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072568653Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072573703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072578573Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072584992Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.07259027Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072596589Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072600048Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072604689Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072608046Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072611402Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:53.072681296Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"87cc436a-ffb5-496f-b3c0-08aa8f720ac3","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized)]","latency_us":527274,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:53.072839479Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.114648175s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87cc436a-ffb5-496f-b3c0-08aa8f720ac3\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87cc436a-ffb5-496f-b3c0-08aa8f720ac3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.18214781Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.182461507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.182471107Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.204310229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.20435271Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.224358528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.224401329Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.244226073Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.244268934Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.263970955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.264009626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.282560499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.282592659Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.30018449Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.300223201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.319772088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.319809859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.338953746Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.338989217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.357778176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.357817817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.376554844Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.376598295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.396601093Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.396639434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.414168833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.414204414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.432298786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.432343907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.450077991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.450110912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.467711612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.467750533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.485683841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.485729543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.503231901Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.503272822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.521159699Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.52120486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.5391866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.539220621Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.556569665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.556608296Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.556835171Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.556949914Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648114632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648175543Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648187894Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648197914Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648207454Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648217854Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648227985Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648238025Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648248225Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648258665Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648268686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648300086Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648310356Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648320397Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648328557Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648338537Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648346188Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648353888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648363328Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:55.648501261Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"eeb6d5e5-b371-4c19-8ba0-c82f8f004c6b","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized)]","latency_us":467488,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:55.648665465Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.491009113s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eeb6d5e5-b371-4c19-8ba0-c82f8f004c6b\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eeb6d5e5-b371-4c19-8ba0-c82f8f004c6b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.134241874Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.134559672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.134566032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.156878385Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.156909776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.175689504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.175723395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.192859824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.192890415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.212543415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.212574575Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.229720265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.229745636Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.248008802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.248044843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.266395391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.266423142Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.283423528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.283450029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.30234249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.302370581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.321552819Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.32159733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.339984289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.34002037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.358188864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.358217035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.377470625Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.377508346Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.395453415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.395486475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.414629833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.414664514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.433927074Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.433970705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.453036692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.453079231Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.470713102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.470740703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.488290032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.488322873Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.505679548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.505713339Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.505936464Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.506061467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593820163Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593853564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593860424Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593865125Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593871475Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593880625Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593885205Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593891345Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593897375Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593903485Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593909686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593915686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593921546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593927766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593933576Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593954427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593960467Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593964827Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.593969237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:49:58.594044769Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"70f6cf2b-ed23-47ba-8380-0edfad3002f6","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized)]","latency_us":460919,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:49:58.594196352Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.155578875s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70f6cf2b-ed23-47ba-8380-0edfad3002f6\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:49:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"70f6cf2b-ed23-47ba-8380-0edfad3002f6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.745452965Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.745696141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.745709071Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.76867191Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.768707751Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.787753666Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.787788407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.80761735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.807658891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.826083811Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.826115892Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.845459864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.845495185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.866106187Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.866140778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.884729412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.884764503Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.903086281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.903129222Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.922296489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.922329449Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.940931215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.940974466Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.961083556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.961124147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.981052843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:00.981103594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.001893131Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.001939182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.022488733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.022532074Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.043633518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.043676959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.064022246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.064072007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.083529661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.083585913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.106172473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.106219663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.125230488Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.125279499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.144044577Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.144098588Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.144294893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.144417386Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241013784Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241081645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241106366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241123946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241132267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241141867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241149157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241167347Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241175308Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241183038Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241192378Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241201428Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241211079Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241217899Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241227019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241236429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.24125014Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.24125834Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.24126566Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:01.241332791Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"44d77fd6-0fda-44dc-88c0-baeb53ee006b","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized)]","latency_us":497166,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:01.241473109Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.376355689s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"44d77fd6-0fda-44dc-88c0-baeb53ee006b\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"44d77fd6-0fda-44dc-88c0-baeb53ee006b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.61550381Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.615813967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.615822458Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.639239997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.639304108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.659624734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.659662855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.678463224Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.678497215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.698867522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.698900102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.716998664Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.717038316Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.734546174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.734588015Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.752249367Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.752286668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.770669837Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.770715228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.789282151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.789330783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.81057726Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.810618481Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.829786269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.82982834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.84868668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.848722601Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.86794204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.867983291Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.88676769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.886810931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.904996255Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.905035476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.924535492Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.924585493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.944487789Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.94452984Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.963694208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.963734519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.984433723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:02.984474444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:03.005419174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:03.005475636Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:03.005691278Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:03.005823004Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105634449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.10567275Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.10568353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.10568803Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.1056925Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.10569822Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.10570165Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105714951Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105720361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105723781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105732391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105736341Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105748842Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105753391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105757582Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105760972Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105764152Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105767712Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105770892Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:03.105808463Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"621499ff-c9f0-490a-a640-b9ecd520717a","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized)]","latency_us":491511,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:03.105976637Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.895264386s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"621499ff-c9f0-490a-a640-b9ecd520717a\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"621499ff-c9f0-490a-a640-b9ecd520717a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:06.667099001Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"364.926133ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:06.99148959Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:06.99190659Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:06.99192233Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.017624295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.017666156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.040570643Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.040616784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.061892252Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.061941963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.082552146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.082600867Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.103549047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.103589652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.122122091Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.122170122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.141433702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.141465187Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.159712019Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.15974914Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.17943876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.179473321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.198713471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.198742971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.216821273Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.216852054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.234819583Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.234858824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.253709895Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.253735685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.272515264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.272550355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.290971705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.290998415Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.310323647Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.310354398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.32884047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.3288723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.347765152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.347794742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.366361596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.366407637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.384740955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.384777726Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.3849724Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.385085013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476466416Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476517107Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476530018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476535818Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476545017Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476550342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476560552Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476573021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476577118Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.4765988Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.47660615Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.4766104Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.47661633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.47662112Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.47662678Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.47663093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476636301Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476647981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476652401Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:07.476701772Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b065c7fa-5bb8-4f22-b8ad-a62ef01563ee","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized)]","latency_us":486339,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:07.476865306Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.015218213s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b065c7fa-5bb8-4f22-b8ad-a62ef01563ee\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b065c7fa-5bb8-4f22-b8ad-a62ef01563ee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.487409797Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.487919329Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.487933839Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.511729628Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.51179096Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.532170616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.534540093Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.551786695Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.551827116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.570083302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.570125643Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.589519496Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.589563738Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.60807816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.610418546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.626812357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.626856769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.645616697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.645679387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.66584941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.665899561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.684722231Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.684763242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.703610702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.703659933Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.722661457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.725023044Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.741085957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.741129228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.75959705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.759644201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.778109592Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.778164053Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.79725765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.7972957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.815756461Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.815810112Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.834510918Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.834547139Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.852812746Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.852852547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.871909132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.871981704Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.872173198Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.872293491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967864564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967933256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967946686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967956766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967964437Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967981297Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.967989747Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968008028Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968015938Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968023658Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968039259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968049079Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968066309Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.96808874Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.96809958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968122301Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968130601Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968138591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968147881Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:09.968201862Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9f30e1e3-d34f-44ee-b0cc-6302cd3e2693","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized)]","latency_us":481925,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:09.968361376Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"6.064985627s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9f30e1e3-d34f-44ee-b0cc-6302cd3e2693\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9f30e1e3-d34f-44ee-b0cc-6302cd3e2693\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.018414634Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.018756448Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.018770418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.040710242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.040754743Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.061295287Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.061336155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.079399357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.079452878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.097842227Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.097883868Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.116935843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.116974204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.134657337Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.134697258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.153899216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.153951407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.175056922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.175113533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.193623865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.193689057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.212084986Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.214487034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.231841388Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.231884669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.249933011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.250005012Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.268048273Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.268091914Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.286350901Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.288677499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.306035491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.306080182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.326179352Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.326212783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.344003088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.344050199Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.362349536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.362413478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.381236707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.381271758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.399627457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.399671118Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.399876773Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.400005786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.494994115Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495086627Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495098285Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495113436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495122668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495133069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495150489Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495160789Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495170269Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495178219Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.49518795Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.49520465Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495224371Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495230541Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495238311Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495245571Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495259652Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495273832Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495282462Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:16.495342974Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7e3c32cc-010a-461c-b17a-028db780352b","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized)]","latency_us":478084,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:16.495508407Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"5.455287617s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7e3c32cc-010a-461c-b17a-028db780352b\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7e3c32cc-010a-461c-b17a-028db780352b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:18.02305214Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62929"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:18.023323086Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"36f9749f-6f46-4d5f-a604-6a93ec8a3f9f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62929","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.937072544Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.937469023Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.937488254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.961466677Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.961525088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.982314215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:21.982371856Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.002156578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.00221314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.021404469Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.021450479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.038959608Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.039008929Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.055049945Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.415366673s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.057696916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.057704616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.076010693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.077929129Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.093248745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.093298686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.111528531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.111581683Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.129774027Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.129819911Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.150604385Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.150653796Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.168331528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.170604083Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.185847317Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.185887738Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.204077633Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.204123814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.222874122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.222915993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.241572478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.24162027Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.259353213Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.259409704Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.278531521Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.278570952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.296322206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.296369042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.314413369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.31445945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.314697825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.314828928Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405612807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405670749Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405694119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.40571142Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.40571946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.40572588Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.40573329Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.40573941Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405752861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405760501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405765311Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405770941Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405778911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405784731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405790001Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405806692Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405814572Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405822352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.405827952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:22.405947335Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a23abe5a-48b8-485a-a948-064701066326","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized)]","latency_us":469997,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:22.406107064Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"5.854998101s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a23abe5a-48b8-485a-a948-064701066326\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a23abe5a-48b8-485a-a948-064701066326\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.24677948Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.247624371Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.247634131Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.270413765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.272827493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.290269839Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.290323051Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.308584287Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.308635558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.327252033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.327289124Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.345036208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.345087479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.363438307Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.363483528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.381514399Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.38156398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.39996242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.400002661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.419725022Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.41978294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.438748809Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.438792067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.45688903Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.456921861Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.475110725Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.475148376Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.493599072Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.493640858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.511625198Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.511677379Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.529821332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.529868793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.548336504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.548378276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.566457337Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.566500578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.584396496Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.584438467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.602847137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.602880507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.620775675Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.620832327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.621113633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.621253623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.71432511Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714401732Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714420522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714431903Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714463013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714474153Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714484404Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714490944Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714499254Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714508054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714516645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714531955Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714542885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714562546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714570586Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714580516Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714588816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714597087Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714606957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:28.71472739Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"06d1f4a6-ae98-412a-ac60-211042d73a1e","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized)]","latency_us":469092,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:28.714882083Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"8.921507401s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06d1f4a6-ae98-412a-ac60-211042d73a1e\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06d1f4a6-ae98-412a-ac60-211042d73a1e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.613482777Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.613815005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.613837784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.637917921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.637976663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.658968654Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.659022785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.67849001Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.678548712Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.697243998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.697292249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.715866343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.715910454Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.735152094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.735197045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.75594994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.756006091Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.775337473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.775389714Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.796315434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.796367862Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.81665033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.816709752Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.837359435Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.837402426Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.859233308Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.859276189Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.879107563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.879155684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.899544821Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.899589602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.919223471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.919271702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.938915651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.938965528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.959874242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.959922493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.980054824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.980097035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:37.99997864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.000027121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.01966096Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.019714871Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.019934007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.02008838Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126857951Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126940843Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126958844Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126967984Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126974604Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126990334Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.126999205Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127007885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127032855Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127051556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127060246Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127073856Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127083117Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127089477Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127098187Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127106577Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127113207Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127121248Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127127628Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:38.12721117Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5cb2637e-f762-4a86-b960-8e5456ae5a03","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized)]","latency_us":514990,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.127380404Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"13.128001699s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5cb2637e-f762-4a86-b960-8e5456ae5a03\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5cb2637e-f762-4a86-b960-8e5456ae5a03\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:38.463830802Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.698758617s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:47.975889801Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:47.975964963Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3f6a85a9-dc97-47b6-8542-ed25162d691b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36105","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":36,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.220709078Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.221587409Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.221595709Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.242779116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.242825786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.262037305Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.262091717Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.280800938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.280863595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.300438833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.300499744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.318496804Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.318550896Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.335649406Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.335701976Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.353153722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.353199543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.371788438Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.371837369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.390687119Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.390746511Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.409713714Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.409777865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.427099499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.4271442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.444749961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.444799982Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.462946145Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.462985216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.481074798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.4811253Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.498944245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.499000387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.516534696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.516587207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.534229818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.534282489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.551736966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.551802638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.569402419Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.56945305Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.586379564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.586450886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.586706222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.586888896Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678596917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678667999Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678682799Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678690119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678695539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678702578Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678726559Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678741138Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678749281Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678756021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678764161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678772032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678776541Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678782902Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678790212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678794692Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678801022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678809272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.678814873Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:50:51.678862614Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c28a7d09-d964-4765-b23e-f51db7523d23","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized)]","latency_us":459351,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:51.679034208Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"9.857193876s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c28a7d09-d964-4765-b23e-f51db7523d23\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:50:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c28a7d09-d964-4765-b23e-f51db7523d23\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:50:56.149615785Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.017137802s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.510441706Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.510843756Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.510866126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.532711408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.532755319Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.553055104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.553105325Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.573667057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.573716018Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.592817424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.592855841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.612488374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.612536685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.630414542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.630463113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.64957504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.649630461Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.667657459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.667703153Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.686394349Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.686464061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.705800373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.705843894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.724845378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.724910379Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.743515524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.743557565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.763562113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.763605414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.783036608Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.78311148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.801490469Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.80153482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.820281478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.822609163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.838723358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.838778159Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.856711948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.856751929Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.875985419Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.876032189Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.894641894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.894701166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.894961342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.895119286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989047329Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989108841Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989119331Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989127262Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989136862Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989144222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989151832Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989164442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989185853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989195443Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989201933Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989209233Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989215834Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989233234Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989240244Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989246854Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989255545Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989261955Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989268195Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:01.989369017Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"195eb89e-8bfe-41f7-8152-0343de26e4c6","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized)]","latency_us":480119,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:01.989567652Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"10.171382653s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"195eb89e-8bfe-41f7-8152-0343de26e4c6\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:51:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"195eb89e-8bfe-41f7-8152-0343de26e4c6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.135361941Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.135940575Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.135958165Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.159144359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.1591962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.179591057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.179639359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.199580355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.199641007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.217168105Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.219024954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.23538327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.235439672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.254780554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.254835365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.272802544Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.272845685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.291999443Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.292040474Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.310916045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.310961906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.330505823Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.330556614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.349718332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.349768333Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.36763368Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.367677931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.386283115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.386337497Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.405420553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.405468364Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.423682829Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.426010064Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.443202135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.443252327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.461435611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.461485262Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.479129514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.479179845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.496843157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.496896088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.514686713Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.514757685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.515082243Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.515265687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608779671Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608840032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608853513Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608859633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608865743Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608877963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608883813Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608895284Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608902094Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608916554Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608927294Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608931895Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608955169Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608960822Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608967807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608979835Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.60898546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608992436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.608999926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:12.609115019Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ee3e3fc7-9d77-4962-a933-47c1dbe49d7c","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\", TimeNotSynchronized), (\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\", TimeNotSynchronized), (\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\", TimeNotSynchronized), (\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\", TimeNotSynchronized), (\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\", TimeNotSynchronized), (\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\", TimeNotSynchronized), (\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\", TimeNotSynchronized), (\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\", TimeNotSynchronized), (\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\", TimeNotSynchronized), (\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\", TimeNotSynchronized), (\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\", TimeNotSynchronized), (\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\", TimeNotSynchronized), (\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\", TimeNotSynchronized), (\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\", TimeNotSynchronized), (\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\", TimeNotSynchronized), (\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\", TimeNotSynchronized), (\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\", TimeNotSynchronized), (\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\", TimeNotSynchronized)]","latency_us":474869,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:12.609276973Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"23.547643907s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee3e3fc7-9d77-4962-a933-47c1dbe49d7c\", \"content-length\": \"124\", \"date\": \"Wed, 04 Sep 2024 16:51:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee3e3fc7-9d77-4962-a933-47c1dbe49d7c\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:13.157287914Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"8.25582496s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:17.929236274Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42093"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:17.929291326Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"44f528d2-0d35-4110-a9be-b029ca5d1eba","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42093","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: a389897c-961c-454c-9a47-71a26222a8f9, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9df9eb77-b91b-4b18-b9ba-51b37aa05a06, kind: Service { id: e5dbb7cb-9b3b-4016-a67c-456d1beabcd1 }, name: Name(\"ntp-e5dbb7cb-9b3b-4016-a67c-456d1beabcd1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 126])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 0a419753-947c-4e72-9631-ad91417fc1de, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b03e478-8f1d-4a36-a394-689d44621766, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9cc6b986-2199-4475-a594-9f92d0733211, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0222011d-8b68-4644-b4a0-426eb4e4ea2a, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5b57b78-0134-4530-960b-cfc349c80b5e, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: e114f27b-91f6-47de-910d-582d0ae0559c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d5788a18-033a-4416-8dc0-5eb6573503ea, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External } } } }, OmicronZoneConfig { id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External } } } }, OmicronZoneConfig { id: 64511683-fa07-4013-ada6-4ab1c3260754, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 58780784-a0f7-44da-a8ef-d1a3dac28453, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External } } } }, OmicronZoneConfig { id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.095071029Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.095373446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.095382276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a389897c-961c-454c-9a47-71a26222a8f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.118800836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.118851307Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: e7df0f2c-2ad1-4f7d-a845-9e8acbeabe56, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.140622047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.140669228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5e402c3a-7e0c-49f4-9e03-e03a9a42f012, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.159917658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.159965629Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b5222f50-3cda-4d9b-9e13-07be3e4b1979, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.179268991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.179310661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 4dec1158-5c1a-4225-8e1d-6b8e2f11748c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.1985175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.198573552Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0a419753-947c-4e72-9631-ad91417fc1de, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.217387961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.217431722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: eeb8af3a-e1a6-41fc-aa51-1a104791b6ac, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.235808391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.235850442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ee268c38-8f2f-4e2c-bc1f-b470afd9fa04, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.254270502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.254313213Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 1b03e478-8f1d-4a36-a394-689d44621766, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.272553919Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.27259523Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9cc6b986-2199-4475-a594-9f92d0733211, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.291490031Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.291548383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d5b57b78-0134-4530-960b-cfc349c80b5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.310549957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.310593318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b39709ed-6d75-4f85-8807-4f24fdad5a5b, dataset_name: DatasetName { pool_name: ZpoolName { id: 62c25a09-dd5a-42bc-bdc0-88462c2034ea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.328232979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.32827857Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b6f9846c-b8e4-4a8c-8f15-0b4557e97c72, dataset_name: DatasetName { pool_name: ZpoolName { id: 9859c4c0-8dca-4c62-8980-22f58b7298bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.347067459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.34711388Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c, dataset_name: DatasetName { pool_name: ZpoolName { id: e5004d09-cef9-4e6f-b2aa-6d7ade3877f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.365279894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.365326005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8, dataset_name: DatasetName { pool_name: ZpoolName { id: f73a9d5e-d846-41dd-90e4-dedfd04db60a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.383413388Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.383457529Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 64511683-fa07-4013-ada6-4ab1c3260754, dataset_name: DatasetName { pool_name: ZpoolName { id: 84deb833-6610-41c6-8b5e-44fa2aeee752 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:36.389928953Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"17.707933838s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.403067377Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.403109218Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71b79b94-57a0-43c7-bfb5-75d1e3399372, dataset_name: DatasetName { pool_name: ZpoolName { id: 41da29c9-1a43-49cc-b472-5dde9cb41a62 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.421097078Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.421140439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 8481469c-ac9f-48e4-b8d1-442a6c730e0e, dataset_name: DatasetName { pool_name: ZpoolName { id: d2efc072-fbdb-4b59-a671-9eb348ded4ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.440842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.440887061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 58780784-a0f7-44da-a8ef-d1a3dac28453, dataset_name: DatasetName { pool_name: ZpoolName { id: 5f5397b7-7011-4692-88c2-8a70902c2fea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.459628258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.459667179Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: fe4f6bc1-98f4-4ca1-b363-fadbc0affd94, dataset_name: DatasetName { pool_name: ZpoolName { id: a145c7dc-0acd-4e57-96ae-74d5f1432797 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.477414863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.477466095Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.477751811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.477913875Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.569527844Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609180841Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609541309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609620341Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609645112Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609663702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.609681943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.625710616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.625743446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.625749177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.780289409Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.81636343Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.816640417Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.816736519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.875959054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.87622827Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.876259981Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.876295152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.876335103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.876818524Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.881023195Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.885115973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.889508668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.894752943Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.900182423Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.905161212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:36.910277774Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.031661384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.03694838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.041872888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.046275553Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.051144319Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.056658937Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.061952958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.067025649Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.072557391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.077413947Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.083190955Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.088909102Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.093670795Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.098792458Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.106208175Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.113455408Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.120997778Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.127969235Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.135803442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.142485571Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.211416238Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.276655777Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.337582362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.397976605Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.460691733Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.462738452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.462761623Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.522363797Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 247, 178]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.522397397Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.526049465Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.526068715Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.582349169Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 220, 19]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.58237742Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.584963692Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.584980332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.640787075Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 245, 247]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.640812806Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.642838304Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.642851735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699476688Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 231, 162]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699504888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699761884Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699790575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699866097Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699878477Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699902527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.699908538Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.70125561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.70126974Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.758807605Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 219, 38]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.758830406Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.759574973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.761000957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.763123158Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.763136178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.763144769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.765779832Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.985841789Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.98591824Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.987339265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:37.998253315Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.018022858Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.037991055Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.038060396Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.038346233Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.041206681Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Installing Omicron zone: oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.041326244Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.053746221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.062712115Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.072075889Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.122594836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.277450975Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.325124634Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.358492972Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:38.389454501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.948222099Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.948455085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.948528606Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.957678175Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.968431872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:39.968771864Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.438225916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.43841096Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.438471201Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.447159459Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.447570779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.4475961Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.45680856Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.457027085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.458594782Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.458887269Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.459096994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.459176316Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.468091289Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.476475229Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.484503091Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.492580664Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.502322964Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.51751508Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.756871947Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:40.757143874Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 53.384759ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.566486589Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 74.910559ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.601528076Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 68.473328ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.635428766Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 50.416438ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.691989287Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 60.727883ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.760862922Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 59.967742ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.796254228Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 105.248324ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:41.847273657Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 212.13009ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:47.303403772Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 62.872411ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:47.42798209Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 129.194542ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:47.799618684Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 264.085169ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:47.983939995Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 45.178481ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.349825802Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 66.16606ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.381387704Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 25.396014ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.511167196Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 33.502865ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.565399057Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 48.762839ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.597153624Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 69.887526ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.633289205Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 34.744155ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.670711986Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 291.364797ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.705818883Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 42.671205ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.733340558Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 112.271221ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.750262062Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 34.2821ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.761156371Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 225.875341ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.772584504Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 873.503061ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.784468267Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 99.177969ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.79596043Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 57.052236ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.81146703Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 128.945061ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.82404497Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 95.510299ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.836063966Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 140.922377ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.847360885Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 53.589293ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.859551805Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 145.565381ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.870606323Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 61.77539ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.887993943Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 191.903156ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.899886226Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 111.089783ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.911871052Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 280.091354ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.922820163Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 165.862171ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.93529866Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 264.9989ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.946095747Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 263.587166ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.955497521Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 287.029687ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.964913126Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 263.340412ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:48.979203126Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 210.376201ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.002680975Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 966.658785ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.017085589Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.19544077s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.03016836Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 154.472861ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.043031077Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 165.371457ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.055668598Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 385.130723ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.108357523Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 226.11672ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.121563808Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:49.190012729Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:49.191921204Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 583.80874ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.215984407Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:49.218748273Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 447.27939ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.231162249Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:49.231270252Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 301.645518ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.251825561Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 544.881771ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.266765797Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 585.878234ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.281864897Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 420.430748ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.296937826Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 361.665974ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.307749414Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 527.736549ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.317649844Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.051133122s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.359947047Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 39.440335ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.472656712Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 30.995451ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.484569836Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 715.220154ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.507737768Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 63.405189ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.523483143Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 132.887463ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.533315488Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 484.626032ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:49.710924979Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 712.12852ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:51.92264022Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.112253914s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.116283724Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 953.56662ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.155968909Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 961.666185ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.296228631Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 174.161544ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.37676267Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.110989092s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.40040885Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 427.244398ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.437375664Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.06574313s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.451304155Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 816.799225ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.467419326Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 173.525103ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.482266063Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.369596565s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.49891482Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 500.932166ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.516051958Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.003786853s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.529468828Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.104720631s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.54214208Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.298561484s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.557088843Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 243.404701ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.572621715Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.141955041s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.651210207Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 542.3397ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.671385228Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 1.138411021s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.833853579Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 635.276724ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:52.878321538Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.183949161s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.030137615Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 881.096547ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.123066619Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 491.982847ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.227862345Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.047218832s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.242149566Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 869.624513ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.256464117Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 562.382888ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.315687958Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:53.374314865Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:53.374340376Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 509.361869ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.526242115Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.211219896s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.536996391Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.403400041s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.604642652Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 56.386836ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.613185066Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.360609489s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.623062391Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 681.82589ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.65830697Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 81.572451ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.679812573Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.405758818s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.732355345Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 166.394538ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.772693736Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.429190703s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.804388511Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.096221716s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.870428984Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.153044227s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.882351978Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.23111508s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.895287547Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 438.664198ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.955755417Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 859.367845ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:53.985881015Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 890.70077ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.017961009Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.408315092s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.048915057Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.063998423s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.637057028Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 747.540887ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.648799528Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.493971283s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.659937663Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.148658193s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.670769831Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 917.518806ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.680474502Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 994.89409ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.761240897Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 938.456902ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.857523591Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 749.712861ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.922203792Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.243030523s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.977722584Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 941.078222ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:54.995816525Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 735.345991ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.021811965Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 836.35022ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.04764063Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 644.644294ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.140439171Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 786.913993ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.151179007Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 913.581433ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.244979851Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 924.550495ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.407727689Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 976.31172ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.469318706Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 608.768534ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.610371366Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.155140664s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.68481034Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 628.174672ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.715501261Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.212936827s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.768788761Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.412398331s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.779227689Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.238630559s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.796296771Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 906.371307ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.80612825Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 753.782387ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.830390438Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 709.177346ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.894500516Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.051429036s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.948350588Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.469903804s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:55.96186575Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.023645743s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.167711975Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 503.933438ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.180315985Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 564.466511ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.23255795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 699.984034ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.244636298Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 877.402101ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.346678948Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 933.868856ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.357812974Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.378350316s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.453016992Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 758.887124ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.595560238Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 543.063874ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.616952048Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 988.791783ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.698789427Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 786.39456ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.725559755Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 893.277574ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.810693033Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 540.706584ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.852082459Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.220519231s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.955849532Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 741.170036ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:56.993317644Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.440049398s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.012561313Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.28873234s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.048713394Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 778.525246ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.173416795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 961.383247ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.205005058Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.11252314s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.218745806Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 847.256254ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.23781967Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.227198227s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.301425755Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.354317956s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.365332368Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 638.663071ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.405450104Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 605.498131ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.441355229Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 775.631507ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.520745301Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.487285205s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.697572553Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 608.524193ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.713266157Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 556.556178ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.743398445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.068822079s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.841444031Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.012885109s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:57.962049514Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 637.059997ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.056248139Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.475411362s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.065460298Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 814.722782ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.094644663Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 936.686699ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.175476269Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.396108217s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.185721884Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 583.092085ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.305752913Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.283724546s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.314150543Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 1.417320123s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.330706888Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 653.61534ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.340163333Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 702.049393ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.348031471Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.426111192s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.461297219Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.065061331s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.53772572Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.220435191s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.703070109Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.162384497s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.727952862Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 1.264739375s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.897173334Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.452234824s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.919707471Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.260791632s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.928434948Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.198014658s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:58.983704995Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 645.602549ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.002597335Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.463773596s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.059551492Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.341464135s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.123365443Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.012182943s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.291915159Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 735.089569ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.552151338Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.459697653s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.590778899Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 534.226511ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.607430645Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 612.807203ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.616401089Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 796.070244ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.663600683Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 1.447840916s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.761222789Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:59.764293533Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:59.764558589Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:59.764705202Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:59.771934115Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 770.52393ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.89717546Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.317343852s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.905720362Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 27.394188ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.957543797Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.221498782s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.965478596Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 62.396829ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.984897058Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 80.221077ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:51:59.993851342Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:51:59.999400264Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:00.002054967Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 119.88652ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.066659366Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 214.989701ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.08903711Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.197824113s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.155475442Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 977.135829ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.173755818Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.399106851s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.192892784Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 225.596327ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.20446738Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.062316852s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.213676159Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.26501433s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.239858573Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.162706859s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.299644117Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 252.267939ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.315344002Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.006996789s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.325509544Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 56.697012ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.337156651Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.18740073s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.380636887Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 147.889958ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.402099998Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 436.670599ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.437390969Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.032145141s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.468670995Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 998.436606ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.477887604Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 975.316463ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.534661447Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 299.677074ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.558042193Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 931.174779ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.576747069Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.048123585s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.688773608Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 206.153366ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.872353202Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 756.438201ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:00.885772331Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 506.852813ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.063631869Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.09030175s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.094408912Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 1.296813074s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.162625168Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 822.995094ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.198241316Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 700.963569ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.219471652Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 972.261878ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.23448644Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 578.982551ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.28695189Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 554.833163ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.343926647Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.091884945s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.364725163Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.181269939s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.52789047Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 653.756735ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.540796218Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 698.386536ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.551783589Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 1.407783907s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.562592346Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.088103113s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.571718244Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 656.01456ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.580194116Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.369553806s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.588393281Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 956.747313ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.596539555Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 734.931751ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.607707381Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 1.005607585s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.654723772Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.213740242s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.749139841Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.08526482s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.878743009Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 529.388156ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.909919452Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 996.872801ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:01.9345883Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 659.516893ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.034713275Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.350533595s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.199586733Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 564.60891ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.214918828Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.004200135s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.229215669Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.283521272s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.252424202Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.088459062s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.265655957Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 766.90236ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.355083458Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.452262362s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.454452535Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.372365613s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.469803145Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 714.856365ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.483813145Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.283351568s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.567620921Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 619.347628ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.673976825Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 508.535629ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.688673495Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 792.634177ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.711762485Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 536.747742ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.725289758Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.387983852s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.798078382Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 856.85424ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.965921701Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 555.885986ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.984524754Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.10507521s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:02.998132058Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 861.326007ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.009509119Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 1.457936455s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.018963604Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.316745509s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.137020817Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 1.486926488s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.215340053Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 916.749104ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.22861954Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.06924739s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.24711341Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 800.752203ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.278639911Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.118730847s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.311354571Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 667.818971ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.37215497Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.48642541s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.519019819Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.257543354s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.548715906Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.30512341s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.563314033Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.305186895s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.577634634Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 1.098522141s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.83893714Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 1.231506224s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.855216428Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.284372872s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.866142308Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 525.146444ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.88089841Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.41189946s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:03.917378799Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 644.776282ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.05047911Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 845.557313ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.089248544Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 610.95721ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.11470195Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 1.477111205s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.157873989Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 799.595935ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.195525566Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 807.918257ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.328996186Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.499699992s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.417635648Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.37356961s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.439683613Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 609.372862ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.462598219Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 1.187774939s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.485630578Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.062689068s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.705949916Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 1.355656318s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.71406582Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.004349636s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.734514017Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 836.18372ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.818985549Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 541.354454ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.881619332Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 537.906164ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.89371746Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.07539485s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.948497715Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 524.8459ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:04.959332503Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 631.379899ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.006706242Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 1.434028947s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.016932325Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.261778285s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.086741809Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 896.404954ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.101139382Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 603.174714ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.150331234Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 1.396126986s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.163784144Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 812.520436ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.344586562Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 781.40092ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.434646648Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.172451987s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.445264541Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e\") failed: Property not found. retry in 1.416754731s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.501232894Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 882.659844ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.649583218Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 692.927421ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.663401107Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.22461322s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.683280011Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 725.270174ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.701752961Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.254487006s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.755240615Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 1.298739506s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.769689139Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.201989305s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.782170107Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.175197196s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.827278062Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.225933841s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:05.933713747Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 974.048616ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.01229075Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 1.051576753s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.0374947Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 691.963351ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.082539563Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 796.200287ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.17347827Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 1.144640421s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.230269933Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 979.399919ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.366604171Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 529.274942ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.379729604Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 1.29522398s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.439309453Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 596.745246ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.464266498Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 616.25296ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.549048098Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 586.570059ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.574883783Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.315302718s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.632725841Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 807.260822ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.789713252Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 1.271883309s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.921182474Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 527.909786ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.932167735Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 1.130338493s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:06.994290566Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.04352155s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.006499497Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 610.41087ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.017613421Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 597.08866ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.031545553Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 773.646743ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.042719549Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 573.470078ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.072212454Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 681.283521ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.082024976Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 866.901654ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.101622453Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 861.379521ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.170217377Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 959.617729ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.179830376Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 772.169793ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.189163799Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf\") failed: Property not found. retry in 1.243052177s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.35544785Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 922.150109ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.38481691Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28\") failed: Property not found. retry in 1.350411567s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.470036081Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 1.236722655s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.606792848Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 628.68103ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.638906883Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 947.914899ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.647273693Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 912.945595ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.655823236Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 1.243466389s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.744741055Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 782.871785ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.773655304Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94\") failed: Property not found. retry in 1.46693911s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.826437551Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 1.211050145s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.957724779Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 548.019228ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.971876857Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766\") failed: Property not found. retry in 873.290807ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:07.979663762Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453\") failed: Property not found. retry in 1.258838274s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.043325169Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a\") failed: Property not found. retry in 1.124649624s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.059440793Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea\") failed: Property not found. retry in 681.736157ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.136270143Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c\") failed: Property not found. retry in 810.503033ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.15039352Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c\") failed: Property not found. retry in 802.135892ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.202080901Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b\") failed: Property not found. retry in 681.756234ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.28012399Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754\") failed: Property not found. retry in 959.616218ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.31663158Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211\") failed: Property not found. retry in 890.567365ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.530886005Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72\") failed: Property not found. retry in 586.667214ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.566984024Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8\") failed: Property not found. retry in 724.191442ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.579173835Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372\") failed: Property not found. retry in 1.143950082s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.605469711Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0\") failed: Property not found. retry in 1.142095821s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:08.865924357Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d\") failed: Property not found. retry in 543.167587ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:09.01087763Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e\") failed: Property not found. retry in 786.275234ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:09.181961767Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:09.596674136Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:09.626245981Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"46.63920485s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:09.631530067Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:09.890007145Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039081877Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_ae7737a4-d65d-4893-87e3-c82d1ff20f28"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039164049Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_clickhouse_d5b57b78-0134-4530-960b-cfc349c80b5e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039173169Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_f0c23dae-e8bb-477b-bc4f-f7bf70b98ecf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039179619Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_64511683-fa07-4013-ada6-4ab1c3260754"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.03919312Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_71b79b94-57a0-43c7-bfb5-75d1e3399372"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.03920723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_oximeter_0222011d-8b68-4644-b4a0-426eb4e4ea2a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039217Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_1a1d03c0-7ef4-4edc-a04a-22115013608d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.03922162Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_e114f27b-91f6-47de-910d-582d0ae0559c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039226271Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_d5788a18-033a-4416-8dc0-5eb6573503ea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039230781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_b6f9846c-b8e4-4a8c-8f15-0b4557e97c72"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039241551Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_65c986e3-6575-46c0-b603-c3cfe853f1c0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039246191Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_ec1adbda-b5d4-49fd-ab62-5ed72a1b8fd8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039250021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_8481469c-ac9f-48e4-b8d1-442a6c730e0e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039254091Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_9cc6b986-2199-4475-a594-9f92d0733211"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039258381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_58780784-a0f7-44da-a8ef-d1a3dac28453"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039262341Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_b39709ed-6d75-4f85-8807-4f24fdad5a5b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039266632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_fe4f6bc1-98f4-4ca1-b363-fadbc0affd94"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039270921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_9d5f56e5-2d2f-486d-9d1c-3ebf0bfe714c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.039275492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_1b03e478-8f1d-4a36-a394-689d44621766"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.048391119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.051625616Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.067866563Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b891c94a-792f-4743-8fc4-e88a925e0b52","remote_addr":"[fd00:1122:3344:101::1]:38800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":34001734,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.089314804Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1286"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.089923358Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.090148344Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.103860051Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.105833418Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n local_pref: None,\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.152592961Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:867"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.15923675Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.277729933Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.397156328Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.600492562Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:10.600848961Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.767500861Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:10.982905143Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:11.263317774Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:11.567749757Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:11.603390086Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:11.808859431Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:11.886315417Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:12.345694222Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:12.607192011Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:12.910679312Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:13.606555481Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:13.610464544Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:13.885913537Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:14.423587067Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:14.614650649Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:15.588781567Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:15.617967582Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:15.885552167Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:16.620909507Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:16.989258363Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:17.623369721Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:17.641172925Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:17.885982027Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:17.99017828Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:17.996812638Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"19243b07-97de-47a5-a287-9bd6d0085247","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42350","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5277,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:18.627081674Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:18.807478682Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:19.630384367Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:19.885560957Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:20.416790764Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:20.633830704Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:21.636716708Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:21.885914185Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:22.640037682Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:23.212530052Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:23.642194538Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:23.886263323Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:24.644735773Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:25.648291623Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:25.88533553Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:26.454772957Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:26.651983256Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:27.654788067Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:27.886124489Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:28.657782974Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:29.661868606Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:29.885896473Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/54308cdc-0331-4daa-84cf-3ba0a405e364/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:30.625599546Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:30.665518577Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:31.669400725Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:52:31.885622296Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d5abb5c3-38c8-4f7b-a191-64252eb83a69): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-04T16:52:32.673434316Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:36.070660674Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69da1d82-053e-48b5-ad7f-685f7aad41c9\", \"content-length\": \"173\", \"date\": \"Wed, 04 Sep 2024 16:52:36 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"54308cdc-0331-4daa-84cf-3ba0a405e364\\\"\", request_id: \"69da1d82-053e-48b5-ad7f-685f7aad41c9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:36.140264902Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.3630383Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.364160497Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8a6f0f25-2b1c-4646-9138-6c1a4274cd90","remote_addr":"[fd00:1122:3344:101::a]:40271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1078,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.446416956Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49038"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.466230059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.591081553Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"84a6fde3-9696-4372-9e30-0a8b2dcaa5fd","remote_addr":"[fd00:1122:3344:101::a]:49038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144524,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.6002168Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.600329363Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.603166981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.603466388Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f84237eb-cbbf-4422-810c-5ae5fd2468b4","remote_addr":"[fd00:1122:3344:101::a]:40271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12054,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.603591431Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"33e030ee-ffcc-4a96-b336-36889e6705df","remote_addr":"[fd00:1122:3344:101::a]:49038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12017,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.77355045Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47894"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.773616491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.912160012Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"f41b895d-f92e-471e-8812-b3a90370f394","remote_addr":"[fd00:1122:3344:101::b]:47894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138502,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.912331407Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46262"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.912389978Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38699"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.913535365Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d4ae1593-25c5-486b-a3ec-79b37e14af8b","remote_addr":"[fd00:1122:3344:101::b]:46262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1153,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.913597697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.913641658Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.913860113Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b707ec8d-8522-452c-9f38-f800a863fc0a","remote_addr":"[fd00:1122:3344:101::a]:38699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1362,"response_code":"200"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.914512038Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.914921798Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"963842ba-5914-4a37-b554-9d66ce02574f","remote_addr":"[fd00:1122:3344:101::b]:47894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2332,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.930353016Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b010340f-7c68-49c4-9076-19d05005a9bd","remote_addr":"[fd00:1122:3344:101::b]:46262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.990339805Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:38.990433977Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0b15465a-e275-4dfa-bc78-c732612be932","remote_addr":"[fd00:1122:3344:101::c]:51889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.003855807Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bbff9437-3b65-492a-8a54-dacd741f3286","remote_addr":"[fd00:1122:3344:101::c]:51889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":122,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.017973933Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39053"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.018947837Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"76806939-e6c3-4cf7-a48e-dff95c68da19","remote_addr":"[fd00:1122:3344:101::b]:39053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":896,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.10094784Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.101131055Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3f4b07ae-7a42-4a9e-ba61-5db499d514d1","remote_addr":"[fd00:1122:3344:101::c]:63117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.834678891Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55592"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.834757613Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.921754116Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"362e4d0b-1133-44bf-8bd2-dd00311bc8aa","remote_addr":"[fd00:1122:3344:101::a]:55592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":86983,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.921858358Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58499"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.922016672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.922157375Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.924203604Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.92571632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:39.926232152Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"09c781a9-1ff7-48b4-bb0c-4e090cc78064","remote_addr":"[fd00:1122:3344:101::a]:55592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4171,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:40.007240142Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"c0feeca3-a8b0-4a26-9de3-aa569ef65418","remote_addr":"[fd00:1122:3344:101::c]:58499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85267,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:40.007706958Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:40.007767879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:40.008252775Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:40.008573784Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"a10fc3c6-d68e-4ea3-bad4-5f17b7ec8d19","remote_addr":"[fd00:1122:3344:101::c]:58499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":965,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.557935507Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39193"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561708267Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561755408Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561778759Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561792339Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561804279Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.561812229Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.56182234Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.56185779Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6a502bb8-6a55-4a9d-8389-a0c4e8347974","remote_addr":"[fd00:1122:3344:101::a]:39193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3849,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.562486505Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:939"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.957777254Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59313"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:43.957880776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.052433059Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"7244d416-4462-4255-a6b3-ef64b221affd","remote_addr":"[fd00:1122:3344:101::a]:59313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94558,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.052838448Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.053050313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.053396181Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.053673808Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"d130601e-9e6d-4584-a0bf-76890eda8dba","remote_addr":"[fd00:1122:3344:101::a]:59313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":932,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056438364Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59662"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.05668991Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056724951Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056759022Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056767672Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056778262Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056784872Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056795993Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.056808853Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3a5518df-fc72-4216-b585-313ac7a793cb","remote_addr":"[fd00:1122:3344:101::b]:59662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":256,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.478852018Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45946"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479055663Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479102654Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479123605Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479136705Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479150115Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479167546Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479172476Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.479182326Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"cbd6e983-a3f8-4fd6-9e8b-ae18aad1f1f9","remote_addr":"[fd00:1122:3344:101::c]:45946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":296,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550651088Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57705"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550852853Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550901514Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550929215Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550945455Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550951595Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550972016Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550977536Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:44.550993787Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"195b01d0-07b6-40ca-9434-7d6dbe1e0bd3","remote_addr":"[fd00:1122:3344:101::a]:57705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":253,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:48.049828306Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:48.050100053Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6afb3210-32d2-48b7-b848-f630c0be043b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59979","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:51.68444663Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:51.685249349Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"453e303a-f662-428f-948b-99c68f3bd47a","remote_addr":"[fd00:1122:3344:101::a]:45945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":775,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:52:51.693394414Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0d9e55f1-c797-4b8e-8d2e-e09ecda781d3","remote_addr":"[fd00:1122:3344:101::a]:45945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.165977821Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50665"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.167142409Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9b352e3d-5e19-404d-a22a-91ea3c5ba60a","remote_addr":"[fd00:1122:3344:101::a]:50665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.186997082Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3952b14f-10ad-4ca0-9880-78f662242201","remote_addr":"[fd00:1122:3344:101::a]:50665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.360456855Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.36110655Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2c726e8d-2378-4c68-8944-ef8766f5bcbd","remote_addr":"[fd00:1122:3344:101::a]:46995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":602,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.471974492Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58877"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.47400078Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.564410504Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.652262607Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f6e8bd1f-f3cf-49d2-8b57-d3a08f23a676","remote_addr":"[fd00:1122:3344:101::a]:58877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":180240,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.655471343Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"36645e96-7ef7-46c3-a711-3ea8664040ea","remote_addr":"[fd00:1122:3344:101::a]:58877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2942,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.72922956Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.731377922Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.73259Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.733562364Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.733728378Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"d39bfb9f-1f2e-477a-9893-350350ea4155","remote_addr":"[fd00:1122:3344:101::a]:58877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6384,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.801217126Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.801516933Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"28eec397-1471-4621-bc00-9ef886d0f5d6","remote_addr":"[fd00:1122:3344:101::b]:49330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":244,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.861375779Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.861511242Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5cccf225-04f9-4bd8-9ff3-33d80d6eb741","remote_addr":"[fd00:1122:3344:101::b]:63958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:08.873526428Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b29d105b-4856-4756-ad59-50f7b68f9cae","remote_addr":"[fd00:1122:3344:101::b]:63958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":861,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.011971317Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48887"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.012124931Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.048546648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.104838649Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"35020624-5cfd-4a0a-96b6-0dcfb99c46c3","remote_addr":"[fd00:1122:3344:101::b]:48887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92784,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.105150027Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33812"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.106061659Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"655e321f-e265-4d0a-bc79-ca5f23436b4f","remote_addr":"[fd00:1122:3344:101::c]:33812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":817,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.106283924Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ecddc7f0-35c7-4ad7-9a6c-8d895024c929","remote_addr":"[fd00:1122:3344:101::b]:48887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.115977245Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d2fd6923-5473-43c2-affd-02b6edf5284b","remote_addr":"[fd00:1122:3344:101::c]:33812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.140173741Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56441"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.140338895Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"75ad8072-4a06-49e7-835c-fa80e5063b6e","remote_addr":"[fd00:1122:3344:101::c]:56441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.32900495Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61566"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.329085662Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.367493587Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.423545293Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"bb73e073-46ce-477d-b92d-f384384efcf6","remote_addr":"[fd00:1122:3344:101::c]:61566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:09.423881241Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"615b3332-16d7-4ea9-b319-47cab8c879d8","remote_addr":"[fd00:1122:3344:101::c]:61566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:18.08415632Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:18.084212982Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"71a8db56-cc07-4485-8ae9-833f6f8ad6f3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37458","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:26.409655235Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:41986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:26.472872411Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"14c85a52-0b4d-472f-8e79-809dfb116491","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":62787,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.387980487Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.388063709Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1d5547b6-b852-4a0a-b2c7-2d46c1d8e71e","remote_addr":"[fd00:1122:3344:101::a]:61754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.410740279Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"77a223e1-7e79-4d1a-b3de-40c9f01959ec","remote_addr":"[fd00:1122:3344:101::a]:61754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.570429883Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.571314904Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e47cdd40-22ca-4f09-8b5e-ad364381eed5","remote_addr":"[fd00:1122:3344:101::a]:39073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.626434408Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46768"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.62654825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.675711542Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.742162155Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"74bc573b-1878-47c4-9cd6-840275996c74","remote_addr":"[fd00:1122:3344:101::a]:46768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115673,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.743346313Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"040ca1bc-0c96-4fc6-bea5-baefc1eaab05","remote_addr":"[fd00:1122:3344:101::a]:46768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":902,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.760602564Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41530"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.760754808Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e9756b9a-18dd-445f-8930-387530908aa8","remote_addr":"[fd00:1122:3344:101::b]:41530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.771984575Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cd8c7c55-326e-4a7c-af42-328362d780db","remote_addr":"[fd00:1122:3344:101::b]:41530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.941544335Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.941598646Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"94d12011-9716-4dbf-bddf-18479034c323","remote_addr":"[fd00:1122:3344:101::c]:33470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":36,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.963838696Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f154336e-88db-4e50-ae65-d5326baf1dc0","remote_addr":"[fd00:1122:3344:101::c]:33470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.999123157Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:38.999293571Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a9c231ae-0242-4b56-a02d-181e5efb5e68","remote_addr":"[fd00:1122:3344:101::b]:50264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.020035225Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33620"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.020101557Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.066574394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.126288807Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ca586fa2-1f01-4215-a261-6f5761ac78f7","remote_addr":"[fd00:1122:3344:101::b]:33620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106212,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.130504967Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"765dfefd-8e58-402c-a24d-52004e3e78da","remote_addr":"[fd00:1122:3344:101::b]:33620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.246960692Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56951"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.247155267Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d52c726c-2fb7-4ab0-9988-504d3791fe2d","remote_addr":"[fd00:1122:3344:101::c]:56951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.257020372Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58617"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.257103574Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.297727411Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.352184959Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"42f4609e-f490-46df-8ac6-4eaa5beb7ee1","remote_addr":"[fd00:1122:3344:101::c]:58617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:39.352460155Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"952d8a39-c2e1-4d61-b8a7-5033d10bfb9a","remote_addr":"[fd00:1122:3344:101::c]:58617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:42.228038386Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:48.118762251Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:48.118861294Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1d9f2c03-b26d-4bf3-bcfd-00684ecd621f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46185","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:53:56.416574066Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"f0ab8848-bb9c-4da5-844a-134ed1d13005","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7348,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.242874926Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.242980328Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"928472cf-3f2b-49d5-b3b3-3642d56651a5","remote_addr":"[fd00:1122:3344:101::a]:44142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.251277056Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5d3b5925-888c-4deb-9a69-3a0812591169","remote_addr":"[fd00:1122:3344:101::a]:44142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.335980094Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61228"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.336059606Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.375924796Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.434200144Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b7511352-191f-4ae9-b79f-15872e60804e","remote_addr":"[fd00:1122:3344:101::a]:61228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.434325718Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43996"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.434580793Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4849d40c-ac58-46e5-a4ab-3bf8a1d87052","remote_addr":"[fd00:1122:3344:101::a]:43996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":202,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.435365512Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"28392af4-9d36-458c-83cb-225e1138b707","remote_addr":"[fd00:1122:3344:101::a]:61228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":887,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.744150188Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52962"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.744835365Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a8547877-f7dc-466a-bb30-fdaaf2b56ada","remote_addr":"[fd00:1122:3344:101::b]:52962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":639,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.756094423Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"333751f6-1354-4ef0-b7c4-297bbb3590b5","remote_addr":"[fd00:1122:3344:101::b]:52962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.821224605Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59535"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.821418549Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ecad073e-edc1-4375-868d-48ee361b7a53","remote_addr":"[fd00:1122:3344:101::b]:59535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.832241507Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60438"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.83234817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.881206614Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.947675068Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"359cc282-fea1-4c4f-9c02-3a75f4833b73","remote_addr":"[fd00:1122:3344:101::b]:60438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115373,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.947998465Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61778"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.948276322Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3da4bbbb-3258-47de-a684-776a77fe8e43","remote_addr":"[fd00:1122:3344:101::c]:61778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":188,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.948340904Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8195a758-8f84-436c-b774-5515b2ef2f20","remote_addr":"[fd00:1122:3344:101::b]:60438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":312,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:08.958541576Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"12ed24fe-c0d4-413f-925f-ee2fe6f9ef24","remote_addr":"[fd00:1122:3344:101::c]:61778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.02876226Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44371"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.029553679Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"cf1d1c66-79db-47cb-a6b7-a914c1656ebe","remote_addr":"[fd00:1122:3344:101::c]:44371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":751,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.049620457Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63150"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.049800791Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.087482019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.143267008Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"508d0f11-30b4-4049-a6c8-49986f1b920d","remote_addr":"[fd00:1122:3344:101::c]:63150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93542,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:09.143588325Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"efa7a0af-7a9a-435e-9ea7-25155ddcebc9","remote_addr":"[fd00:1122:3344:101::c]:63150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:18.15114492Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33566"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:18.151220842Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"03c383ad-60b4-4c46-809b-33bd3daf6c0f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33566","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:26.41643969Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"319fbeb5-8c55-442e-acf5-4bd4351be3ae","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6775,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.306609731Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.306717994Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d4dc8531-0f18-498f-9411-50c835612a80","remote_addr":"[fd00:1122:3344:101::a]:46009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.321491426Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c7ee8aa4-65c8-4aa6-9e1a-2e5cd125453a","remote_addr":"[fd00:1122:3344:101::a]:46009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":122,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.636646174Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.636859789Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a2330959-6559-442c-adfc-4f8c900057b7","remote_addr":"[fd00:1122:3344:101::a]:45977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.775697507Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.775870961Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"61473167-1858-4393-bde1-e34bfca01507","remote_addr":"[fd00:1122:3344:101::b]:62539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.787764815Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"81515c22-f76d-4ab0-a950-11ef3508e1d0","remote_addr":"[fd00:1122:3344:101::b]:62539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.859217247Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65336"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.85931957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.908607464Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.9743293Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"10495313-224b-469b-b8b8-e9c7019edbca","remote_addr":"[fd00:1122:3344:101::a]:65336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115057,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.979340899Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59497"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.97937831Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"176eb2b9-0996-49bd-bd78-c5587eff5b88","remote_addr":"[fd00:1122:3344:101::a]:65336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:38.980563698Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a2799789-6f0d-4a48-8e4b-9f4a38d73674","remote_addr":"[fd00:1122:3344:101::b]:59497","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.037443343Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63604"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.038206512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.079011884Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.141991554Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9baa4a41-bed9-436d-b49c-383f009aeb08","remote_addr":"[fd00:1122:3344:101::b]:63604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104450,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.142167878Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.143305125Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fd07eba9-eabc-4e12-8ff9-6b8f4914ecfb","remote_addr":"[fd00:1122:3344:101::c]:46179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1086,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.143529581Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9530bbb9-2432-4ca0-8c62-8f2333cde11a","remote_addr":"[fd00:1122:3344:101::b]:63604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.155853564Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"82236c6d-7b56-40c2-ba33-f71ae9c9ba9c","remote_addr":"[fd00:1122:3344:101::c]:46179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.277034452Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47299"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.277120654Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.312062896Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.368848849Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ffc16205-b412-4e9d-8786-e799bc3457f9","remote_addr":"[fd00:1122:3344:101::c]:47299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91768,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.368960452Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.369650698Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f7a8106b-d575-4acb-9b01-f4a47f736874","remote_addr":"[fd00:1122:3344:101::c]:63615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":655,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:39.369779132Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d985d6d0-a1a7-44cc-b92b-44ba9ae8ccba","remote_addr":"[fd00:1122:3344:101::c]:47299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:48.184897061Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:48.184953922Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b3cd7ba4-7f93-4bfa-a526-dc0bca9b3c37","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36931","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:54:56.41658566Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"b5228c13-da18-4771-85d2-2b38ea206cd4","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6698,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.370175793Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.370248935Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"880ca263-cf90-43eb-962b-ab18fd716a6f","remote_addr":"[fd00:1122:3344:101::a]:42726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":54,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.379394453Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"aa8bc6fc-fa07-4a6d-8cfe-6f26588f15bc","remote_addr":"[fd00:1122:3344:101::a]:42726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.520740091Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59388"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.520936615Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1148f459-4727-4927-8390-95848302a5eb","remote_addr":"[fd00:1122:3344:101::a]:59388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.537978561Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62473"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.539154289Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.603380629Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.682595456Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"911af552-0335-4fdf-a411-5650721eb324","remote_addr":"[fd00:1122:3344:101::a]:62473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144532,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.6836076Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3eb2a4db-cb00-48de-b281-414e1b3421d4","remote_addr":"[fd00:1122:3344:101::a]:62473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":740,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.755946714Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.756088578Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e7e6fa71-defa-4e03-8301-b511d0d877db","remote_addr":"[fd00:1122:3344:101::b]:56550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.765307837Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8a3d8a20-5521-4a2a-b054-2562a7f6b0bd","remote_addr":"[fd00:1122:3344:101::b]:56550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.873241549Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58401"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.873416373Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"13a7aa46-be6e-4a48-aad2-08b97b36b1c1","remote_addr":"[fd00:1122:3344:101::b]:58401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.898699425Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39033"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.898782997Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:08.941599067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.006997036Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0aa59699-9ac1-44f9-9b01-7441d8eaea64","remote_addr":"[fd00:1122:3344:101::b]:39033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108246,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.007218141Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.007514998Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fc0f778d-ecbd-4bba-bd95-52f79e8ddca6","remote_addr":"[fd00:1122:3344:101::b]:39033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.007562839Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"20853175-59cd-438a-bc36-568529cc4303","remote_addr":"[fd00:1122:3344:101::c]:47403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":196,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.019956484Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"79cd47fd-8576-41de-bb91-a4de85abac9c","remote_addr":"[fd00:1122:3344:101::c]:47403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.072956057Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40389"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.073698235Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.114524827Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.171888394Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a9f0bd26-7cb8-4859-90e0-644e0946186a","remote_addr":"[fd00:1122:3344:101::c]:40389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98895,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.172182551Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.172456638Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c06cdd6b-7cfe-4663-8ba2-f64d58513965","remote_addr":"[fd00:1122:3344:101::c]:40389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":264,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:09.172575761Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2435aa13-c1a6-4d29-aea9-b9782718ad08","remote_addr":"[fd00:1122:3344:101::c]:50248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":290,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:18.218901068Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:18.219001851Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ffcf4d83-2b39-45cd-b4b0-9e34efd9b211","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54994","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:26.427549409Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"f8ae9c7f-bbe4-4aa0-9385-b1ad050131eb","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5422,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.371609705Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.371717008Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c1dd42d4-6542-463b-9113-f7d19dda9aed","remote_addr":"[fd00:1122:3344:101::a]:40219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.386317315Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"077a24e5-ed91-4db0-bb9a-68802d391ec6","remote_addr":"[fd00:1122:3344:101::a]:40219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.560545096Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.560769361Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e584a6c9-5666-48b1-8693-65cfc5817ebd","remote_addr":"[fd00:1122:3344:101::a]:47694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.635608854Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38036"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.636979377Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.690283677Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.764741681Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"431eb2c7-5f44-455f-9676-db89a4331bde","remote_addr":"[fd00:1122:3344:101::a]:38036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129063,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.765067659Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"941b4f0d-4e18-4e8f-acdf-08dc4ee67931","remote_addr":"[fd00:1122:3344:101::a]:38036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.866641259Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.867804196Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c5665e9e-5d3a-421d-b812-581b90046670","remote_addr":"[fd00:1122:3344:101::b]:62380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.997149118Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:38.998342177Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"78863c17-12cd-4a07-a1c7-7877f87c5bb8","remote_addr":"[fd00:1122:3344:101::b]:63320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1099,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.000115419Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1787ce4e-3d53-4e54-8952-6fa08a4377a4","remote_addr":"[fd00:1122:3344:101::b]:62380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":82,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.099598739Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47545"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.099758813Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.134938691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.196456997Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f548cee4-126a-472a-9417-ae9cc6feb0cf","remote_addr":"[fd00:1122:3344:101::b]:47545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96791,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.196774464Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.197723287Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a110f51d-7abe-49aa-8b1d-1d7bdee9a285","remote_addr":"[fd00:1122:3344:101::c]:41098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":820,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.197774798Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"da309e8b-dfff-47c3-b95e-adbfc718bdeb","remote_addr":"[fd00:1122:3344:101::b]:47545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1041,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.290774114Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.290959378Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5c7a9752-722d-43d2-946b-b170fb8a443e","remote_addr":"[fd00:1122:3344:101::c]:38066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.298883667Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d269c6db-b234-4aa1-9542-243bae2bdd2d","remote_addr":"[fd00:1122:3344:101::c]:41098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.361734184Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55260"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.361796666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.40183373Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.459303169Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f29a18ec-457f-4061-b1eb-f83e9f629105","remote_addr":"[fd00:1122:3344:101::c]:55260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97520,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:39.459566846Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0493d135-9ec2-48af-91d0-c613ddd8b627","remote_addr":"[fd00:1122:3344:101::c]:55260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:48.26148331Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:48.261628514Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"136e20ff-4ef5-43b2-94cd-835af73acb5a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56987","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:55:56.496402487Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"a4e2f9a6-fad0-48f5-b899-253a44a49b79","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6782,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.291065663Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.291140695Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8ab8cbc2-cee7-4353-b636-838bb0ee3b3a","remote_addr":"[fd00:1122:3344:101::a]:55697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.300674652Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"783517c6-a6fc-44c3-b47e-dc1974edbc09","remote_addr":"[fd00:1122:3344:101::a]:55697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":688,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.387194073Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34623"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.387351087Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"465f551e-aa5f-4908-a516-0e5eef7b8dfa","remote_addr":"[fd00:1122:3344:101::a]:34623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.435279219Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54020"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.435364591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.474949644Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.536818458Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"91612880-572f-469a-954a-490c427de976","remote_addr":"[fd00:1122:3344:101::a]:54020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.538489818Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"03b2cbf7-01cc-4096-b88b-4c582c613614","remote_addr":"[fd00:1122:3344:101::a]:54020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1401,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.740942531Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58790"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.741039313Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"42ad27dd-dce5-474f-8ab7-2413b4ceadf8","remote_addr":"[fd00:1122:3344:101::b]:58790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.749980146Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"735572cf-7978-4017-b1a8-3a2ab52cfef9","remote_addr":"[fd00:1122:3344:101::b]:58790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":742,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.910346047Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64497"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.910508811Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"fef62065-60d8-452f-bfda-b0dc8b865966","remote_addr":"[fd00:1122:3344:101::b]:64497","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.94867899Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60402"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.948765142Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:08.987241319Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.049901872Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6c4463e9-797e-4c37-94fa-d1bdf2cb47e0","remote_addr":"[fd00:1122:3344:101::b]:60402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101172,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.050120197Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.050363873Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1d796bd0-67dc-4bb5-9b6e-5b1827dea73e","remote_addr":"[fd00:1122:3344:101::c]:40063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.057099224Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"67de9558-6921-41ab-8e29-d09cb82435a8","remote_addr":"[fd00:1122:3344:101::b]:60402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":792,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.122131383Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.122305547Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"00c993bf-3364-47fd-9718-9836bb6be09d","remote_addr":"[fd00:1122:3344:101::c]:43271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.12828578Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3d95d3d6-50e6-4269-913a-dce9babb8de7","remote_addr":"[fd00:1122:3344:101::c]:40063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.158420387Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64862"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.159076893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.200216893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.261660057Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"41f48441-752b-4dec-8628-5bd431bfba5c","remote_addr":"[fd00:1122:3344:101::c]:64862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:09.267026065Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"39f95c2e-47ae-4474-bd93-357bccfd5cdc","remote_addr":"[fd00:1122:3344:101::c]:64862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:18.296117932Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:18.296179494Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"54044f18-25cb-4296-9c68-65e1ae79c316","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37140","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:26.4995407Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"cbdb8568-23bb-40d9-bee7-0e8b1d750bdf","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7746,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.411610287Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.41170369Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f060b4bf-9778-4d2d-8d7f-f7b6827b8a36","remote_addr":"[fd00:1122:3344:101::a]:37268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.422669591Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3894ad29-45c0-4db0-ba67-c646b266f1c7","remote_addr":"[fd00:1122:3344:101::a]:37268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":797,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.754125247Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47606"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.755014878Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c5bb95b6-6181-4fce-b04d-eb6afb88b705","remote_addr":"[fd00:1122:3344:101::b]:47606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":860,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.826475621Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.826663765Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"21a2c856-7038-448e-8c7f-877c69925df9","remote_addr":"[fd00:1122:3344:101::a]:54223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.88433953Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37805"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.884449882Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.934116145Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.999433732Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3b66e1b3-6d68-4a90-a9a7-03e60c99409d","remote_addr":"[fd00:1122:3344:101::a]:37805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115033,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.999583305Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.999745989Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"58da20d2-3ab7-44b4-86fd-85bcc198df1e","remote_addr":"[fd00:1122:3344:101::b]:55208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:38.999947084Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b1412fb3-5176-4483-b10e-6765b2ced334","remote_addr":"[fd00:1122:3344:101::a]:37805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":198,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.009556133Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1e0857ce-0bef-4a73-93ca-e63f89ad9b3a","remote_addr":"[fd00:1122:3344:101::b]:55208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.041405122Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.041503994Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c54ae96a-f2ef-47df-a697-07e42741246b","remote_addr":"[fd00:1122:3344:101::c]:43617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.052514977Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"011c1222-188f-4924-84a5-bd655125491c","remote_addr":"[fd00:1122:3344:101::c]:43617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1082,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.056526682Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54279"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.056615504Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.096656598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.154435785Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4f32d170-634d-45e6-9ff6-1390f74dc176","remote_addr":"[fd00:1122:3344:101::b]:54279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97863,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.154911306Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3e576a0b-7c05-4866-b67a-94200d1abb04","remote_addr":"[fd00:1122:3344:101::b]:54279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":220,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.272465147Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.273382459Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b7cb8b06-2cfe-45ae-949a-a6b8a8a7631b","remote_addr":"[fd00:1122:3344:101::c]:48950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":888,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.36746555Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58092"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.367554542Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.407807551Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.461973482Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"51e0dcb4-724b-47fd-9e16-5fe5f3f5f8c0","remote_addr":"[fd00:1122:3344:101::c]:58092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94436,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:39.463026047Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5417d6f8-41c6-458d-9388-8d469d870b98","remote_addr":"[fd00:1122:3344:101::c]:58092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":760,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:48.330456793Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63918"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:48.330518194Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e10e99ad-5797-45c3-83ac-4a484fa47240","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63918","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:56:56.498965017Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"2ddd7d0c-7cfb-442b-970b-ed3d8fc1010c","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.236049323Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.236134875Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d781d081-f731-40b4-8c75-85427b581e8f","remote_addr":"[fd00:1122:3344:101::a]:49927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.244865543Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d7d261c1-8af2-485e-a834-ac4f6e3e72cd","remote_addr":"[fd00:1122:3344:101::a]:49927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":870,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.313038147Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58265"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.3131523Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3ad790fc-41cd-4af5-abef-3b9ccff15f85","remote_addr":"[fd00:1122:3344:101::a]:58265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.359775901Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43692"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.359847032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.397442288Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.451744772Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e95afa6f-d25a-4128-a823-f33d7dfdf4f6","remote_addr":"[fd00:1122:3344:101::a]:43692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91928,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.452061009Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b4b09cfe-0221-463b-84fc-5ad1c2e7ae32","remote_addr":"[fd00:1122:3344:101::a]:43692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.793383111Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.794628451Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e66b5d1c-b741-4fe3-ba7c-c81732f827de","remote_addr":"[fd00:1122:3344:101::b]:34442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.944211045Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.944287396Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"af096875-ae05-425b-8207-4ebee247c8bb","remote_addr":"[fd00:1122:3344:101::b]:57003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.952943733Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"37dde79c-4e7c-451d-a05d-e249d1d6034d","remote_addr":"[fd00:1122:3344:101::b]:57003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.960182405Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58030"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.960280108Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bbcc17f0-61a8-4daf-930d-8f238143085f","remote_addr":"[fd00:1122:3344:101::c]:58030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":24,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.971381152Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8faf5735-fda1-43c4-9ca5-d3630b0d37dc","remote_addr":"[fd00:1122:3344:101::c]:58030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.979075185Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59102"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:08.979164167Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.026127156Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.093932002Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"da507e54-a521-4ced-874c-2aae428107d2","remote_addr":"[fd00:1122:3344:101::b]:59102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114802,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.094315031Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c337d3c6-7244-4ebc-abcf-615e3a69b5be","remote_addr":"[fd00:1122:3344:101::b]:59102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.129259714Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58599"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.129388466Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.165836025Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.225843405Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e4bbf4a4-24de-4021-a0a1-cc5a5095603a","remote_addr":"[fd00:1122:3344:101::c]:58599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96522,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.2260568Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60065"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.226289425Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bb7c0c6c-77f7-40e8-abc6-0e799e26dcf1","remote_addr":"[fd00:1122:3344:101::c]:58599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:09.226425318Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2143412e-b958-4a71-a4c0-e8dcc227368a","remote_addr":"[fd00:1122:3344:101::c]:60065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":263,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:18.364097994Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:18.364165205Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"50b20759-f1b5-473f-9fc3-674ec4412247","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62280","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:26.498741291Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"d71acf47-9ffc-4b2d-beb4-e6205bf2751c","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6748,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.545120886Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.545344031Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"50183886-324a-4d64-8cb1-0d52bcbdbeb8","remote_addr":"[fd00:1122:3344:101::a]:55449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.566222588Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.56632047Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e13dbfdb-b548-4406-ad4b-3bc449ff48d7","remote_addr":"[fd00:1122:3344:101::a]:55777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.579908874Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"144a3638-e7bc-40c9-b6d6-f577efde8d22","remote_addr":"[fd00:1122:3344:101::a]:55777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":164,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.656220192Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56183"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.65738715Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.698398487Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.766123691Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c8c60572-ed58-44db-bd19-16401feb1b06","remote_addr":"[fd00:1122:3344:101::a]:56183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109853,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.766440938Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.766755286Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fdb3c8a6-065c-4720-8df8-ce5eaa0d31e9","remote_addr":"[fd00:1122:3344:101::a]:56183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":267,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767601246Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767668928Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767718049Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767747799Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.76778067Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767816031Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767834452Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.767888423Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"89c79a31-aa11-4289-b398-00b1a8fa443f","remote_addr":"[fd00:1122:3344:101::a]:36240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1298,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.857892627Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.858099482Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"91c3ba88-1156-4b06-a796-ebcf6eabdad1","remote_addr":"[fd00:1122:3344:101::b]:60226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.869413022Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9b5f4353-3a84-4268-86ef-0a5a7849798a","remote_addr":"[fd00:1122:3344:101::b]:60226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":154,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.9650468Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:38.965233915Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ff7f14ab-11f6-48c3-9b72-4b701f8bb0f8","remote_addr":"[fd00:1122:3344:101::b]:46160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.042805403Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64099"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.042917166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.088627605Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.150403916Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e9e26f0e-dacf-40ce-b997-42e2ed7ef5bc","remote_addr":"[fd00:1122:3344:101::b]:64099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107540,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.150581701Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46342"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.150693483Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41710"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.15097339Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151044522Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151082793Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151127934Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151175505Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151205286Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151230806Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.151249497Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b9a7b67b-ec0c-49fd-bd44-fd78e79c295b","remote_addr":"[fd00:1122:3344:101::b]:46342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":582,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.152016945Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"eda33eef-150a-422d-9aec-02ed40a5f887","remote_addr":"[fd00:1122:3344:101::c]:41710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1200,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.152134798Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"55cccf13-a54a-47b5-ba3c-858a003276dd","remote_addr":"[fd00:1122:3344:101::b]:64099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1345,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.210757725Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49426"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.210962809Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c9505c8b-eda0-47ce-a5a6-ab6e56764615","remote_addr":"[fd00:1122:3344:101::c]:49426","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":159,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.222223148Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"01db9ffe-ef63-485d-888a-a725c0ae5fd7","remote_addr":"[fd00:1122:3344:101::c]:41710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.346159091Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51741"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.346243922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.387759482Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448347365Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"33588285-129d-4e9d-82fc-92ddd658b60a","remote_addr":"[fd00:1122:3344:101::c]:51741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448482888Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58862"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448838627Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 14b4f63c-5440-46c5-9830-061164264d94, kind: Service { id: 1b03e478-8f1d-4a36-a394-689d44621766 }, name: Name(\"external-dns-1b03e478-8f1d-4a36-a394-689d44621766\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 55605e3e-1625-4b34-bab8-521c907eef5c, kind: Service { id: 9cc6b986-2199-4475-a594-9f92d0733211 }, name: Name(\"external-dns-9cc6b986-2199-4475-a594-9f92d0733211\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 38])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 04756493-5732-48dc-bc05-c507fda7eae9, kind: Service { id: 1a1d03c0-7ef4-4edc-a04a-22115013608d }, name: Name(\"nexus-1a1d03c0-7ef4-4edc-a04a-22115013608d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 247])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 633bff4c-26e8-4a5d-9003-f40bdef6bef2, kind: Service { id: 65c986e3-6575-46c0-b603-c3cfe853f1c0 }, name: Name(\"nexus-65c986e3-6575-46c0-b603-c3cfe853f1c0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 91f4af4b-89ad-4610-8ef8-022ea68633ed, kind: Service { id: ae7737a4-d65d-4893-87e3-c82d1ff20f28 }, name: Name(\"nexus-ae7737a4-d65d-4893-87e3-c82d1ff20f28\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 178])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448892478Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448908759Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"dbd2714e-19f7-4b7b-8b15-e0ab948d4754","remote_addr":"[fd00:1122:3344:101::c]:51741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":252,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448939949Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.44896186Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.44897544Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.44898709Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.448998561Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:39.449016731Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e9c0f117-9492-4f6d-bd1b-fe63b2dad46b","remote_addr":"[fd00:1122:3344:101::c]:58862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":444,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:48.398089433Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54463"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:48.398713488Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"56e1e993-ccd9-4970-a899-4cc9c9739043","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54463","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":598,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:57:56.499264602Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"ca6d7141-3ef0-4250-83d9-ca2651f4d3b3","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.272935302Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.273550606Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"011b13bf-3afc-4abd-b805-4750531e0224","remote_addr":"[fd00:1122:3344:101::a]:50300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.281304841Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7a122a82-92cb-4b72-accb-370d5d7b616e","remote_addr":"[fd00:1122:3344:101::a]:50300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.341570727Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.342366596Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3466eded-3ae5-41bc-bec5-ca6d88f599b5","remote_addr":"[fd00:1122:3344:101::a]:36774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":725,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.429905772Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51590"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.430031135Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.468531332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.523608164Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9017f5e5-d885-4e0e-8988-c0e876f982a2","remote_addr":"[fd00:1122:3344:101::a]:51590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93648,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.52384776Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0f04406b-f4ea-4744-a81b-05c7f9380ef4","remote_addr":"[fd00:1122:3344:101::a]:51590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.727661715Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35609"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.727828499Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0422a134-4eb7-429f-a106-53ad25d37e62","remote_addr":"[fd00:1122:3344:101::b]:35609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.736894315Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6d11892c-5cd7-45c6-9688-6eed73115c1b","remote_addr":"[fd00:1122:3344:101::b]:35609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.830424674Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55965"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.830650979Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a6deb533-8e05-4c52-9e6c-324da2fe9933","remote_addr":"[fd00:1122:3344:101::b]:55965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":187,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.918490882Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33083"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.918579394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:08.95870072Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.02249428Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7bc6cf5f-be89-4e85-8758-4ae825b22cc1","remote_addr":"[fd00:1122:3344:101::b]:33083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103954,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.022740596Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.02291972Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e55b59ab-739e-4297-b284-61fa0b20145b","remote_addr":"[fd00:1122:3344:101::c]:61983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.023837652Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"54908729-8880-495b-9146-18784067d50e","remote_addr":"[fd00:1122:3344:101::b]:33083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":940,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.032224472Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8fbd3dac-d8b3-4ee6-9ded-c56012f0191e","remote_addr":"[fd00:1122:3344:101::c]:61983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.102699241Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.102901276Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f9decca7-23f3-42f6-bf9f-f1bfd9d296a4","remote_addr":"[fd00:1122:3344:101::c]:60034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.105812295Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50075"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.105920357Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.143422911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.200724846Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"27f54ecd-94c2-4e77-9cf0-40b4c6aed582","remote_addr":"[fd00:1122:3344:101::c]:50075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94845,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:09.201023353Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0acbe3c0-6fad-4aa8-b0fd-56e3778022a4","remote_addr":"[fd00:1122:3344:101::c]:50075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:18.432279757Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:18.432350318Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a74a1236-7ecd-4882-a910-507729b80f87","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47723","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.48193414Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.482094904Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9873a7ee-43f0-478b-a0d1-d1578bbafb2f","remote_addr":"[fd00:1122:3344:101::a]:51533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.483497637Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15180582) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.483532728Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"9b774081-55b7-4403-92c1-4e09f9c43a1c","remote_addr":"[fd00:1122:3344:101::a]:51533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.659294515Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53886"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.670676226Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:570","metadata":"InstanceMetadata { silo_id: c0672ccb-368c-4e5d-9ccf-3b69eaaade11, project_id: 31c13375-38d1-467b-91f3-1a998faa0d44 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-04T16:58:20.657178744Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(4188b9f4-4179-44e5-8abb-b1456b39732c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-09-04T16:58:20.212170022Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15180582))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-2cbd5be624b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2dcc5913-e4aa-440a-9814-23467041502a, read_only_parent: Some(Volume { block_size: 512, id: 0be961b2-734e-45bb-9eaa-b9752c22bb25, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c0a78711-284d-497d-8714-38f94f23bde2, key: Some(\"cpPfp5ZKxzPwe7FUIwgRBL3MeSEGn728+8afdQj8TLo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2dcc5913-e4aa-440a-9814-23467041502a, key: Some(\"D6eB1CaLjf/d31jX48AnhYcX/us083RmO91ydfbiAM0=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(..) }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.671173958Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.671633359Z","hostname":"buskin","pid":1562,"propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:991","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15180582))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8d70875-cf69-4d30-95ec-676e5216883c, kind: Instance { id: a35778a4-f10e-4179-a3f6-ac70bd84565c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 140, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15180582), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-2cbd5be624b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2dcc5913-e4aa-440a-9814-23467041502a, read_only_parent: Some(Volume { block_size: 512, id: 0be961b2-734e-45bb-9eaa-b9752c22bb25, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c0a78711-284d-497d-8714-38f94f23bde2, key: Some(\"cpPfp5ZKxzPwe7FUIwgRBL3MeSEGn728+8afdQj8TLo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2dcc5913-e4aa-440a-9814-23467041502a, key: Some(\"D6eB1CaLjf/d31jX48AnhYcX/us083RmO91ydfbiAM0=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-04T16:58:20.657178744Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.673454732Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c","method":"PUT","req_id":"4584b5ea-f44b-41ca-aa21-c206173c48fb","remote_addr":"[fd00:1122:3344:101::a]:53886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":14102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.779279023Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.78124084Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c/state","method":"PUT","req_id":"4f361b12-3ad6-4e72-bf5b-a04f5eb1a0a0","remote_addr":"[fd00:1122:3344:101::a]:34098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1932,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.782221543Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 140, 66]), slot: 0, vni: Vni { inner: 15180582 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.78289789Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.823406495Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:20.8462839Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:22.458340497Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:22.458567843Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 28.702193ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:22.638915849Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 93.69235ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:22.677213981Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 270.65215ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:22.790866589Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 408.824773ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:23.0796727Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 904.681411ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:23.503451157Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 560.887777ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:24.420319981Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 683.836743ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:24.992394601Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 516.266899ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:25.687898451Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 1.399525239s","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:26.215829789Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:26.585984828Z","hostname":"buskin","pid":1562,"uri":"/d5abb5c3-38c8-4f7b-a191-64252eb83a69","method":"GET","req_id":"1dfdc0e9-105a-473f-9696-a7830c2e947d","remote_addr":"[fd00:1122:3344:101::d]:41986","local_addr":"[fd00:1122:3344:101::1]:62744","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8116,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:27.646913294Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1449"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 28.443294ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:27.657052505Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:27.665524657Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:27.665544758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 78.127456ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:27.700112601Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:27.722128216Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.9999981s","next_cleanup":"Instant { tv_sec: 46692, tv_nsec: 433602607 }"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 179.69313ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:27.788456946Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\") failed: Property not found. retry in 421.717868ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:27.979062738Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.412223788Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1458"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.45053434Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1482"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.594038989Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-09-04T16:58:28.600541854Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.600572554Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.600683057Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-09-04T16:58:28.600541854Z }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.601148438Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-09-04T16:58:28.600541854Z }, migration_in: None, migration_out: None }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-09-04T16:58:28.621563495Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.621577465Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.621591215Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-09-04T16:58:28.621563495Z }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:28.621609716Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-09-04T16:58:28.621563495Z }, migration_in: None, migration_out: None }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.412625676Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.412706328Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9e738c72-715f-43b9-8e9e-b92ffa29cc30","remote_addr":"[fd00:1122:3344:101::a]:54550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.484234062Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.485570164Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c/state","method":"GET","req_id":"8b8c489c-25d7-416b-9fc0-080461db2c84","remote_addr":"[fd00:1122:3344:101::a]:61922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1292,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.565255672Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.565525088Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"54721afc-c70e-45e4-8d79-39da535e4724","remote_addr":"[fd00:1122:3344:101::a]:58166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":214,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.66379133Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"02b02a26-e0eb-4bae-99a5-25557239ac85","remote_addr":"[fd00:1122:3344:101::a]:54550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1093,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.715999874Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46176"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.716076866Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.756235202Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.856851689Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0d877a04-3b7d-4da3-8448-502e03c3ee08","remote_addr":"[fd00:1122:3344:101::a]:46176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140786,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.857063545Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.857225569Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5370a8b9-1dde-49ff-b046-5d2a46dd097d","remote_addr":"[fd00:1122:3344:101::b]:37015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.857392493Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"592b57b5-f9f6-4aa2-8684-ab3d14e42b74","remote_addr":"[fd00:1122:3344:101::a]:46176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":212,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:38.884369495Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1dd8fc91-d7b5-4a2f-ae6a-fb38ec4ed051","remote_addr":"[fd00:1122:3344:101::b]:37015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.020939479Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.02180976Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"25e40c0a-9a8a-4005-be09-35c272fe4876","remote_addr":"[fd00:1122:3344:101::b]:63140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":844,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.023343446Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.023477999Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c/state","method":"GET","req_id":"afd172bd-07ea-4015-ae73-0508fbe92af9","remote_addr":"[fd00:1122:3344:101::b]:61912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.083062939Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40402"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.083197162Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.124604549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.182259042Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"53a757f7-7b2c-4666-92f5-5b2426733389","remote_addr":"[fd00:1122:3344:101::b]:40402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.182589401Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50331"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.182602871Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.182921618Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"878a1f41-9ee9-454b-850b-e00a144bd761","remote_addr":"[fd00:1122:3344:101::b]:40402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":260,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.183969163Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e454ed55-1d1f-471e-aca4-2f0923048301","remote_addr":"[fd00:1122:3344:101::c]:37171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1222,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.183980394Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c/state","method":"GET","req_id":"f1564ddc-e7fa-438d-bfde-4729c205b412","remote_addr":"[fd00:1122:3344:101::c]:50331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1291,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.284370655Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61627"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.28455518Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e7c65fae-d308-4c67-9b09-9dd7eedf51a5","remote_addr":"[fd00:1122:3344:101::c]:61627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":160,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.292795086Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"52078367-a7e9-4975-aae0-2eb6c46913ed","remote_addr":"[fd00:1122:3344:101::c]:37171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":658,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.34877575Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62842"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.348873682Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.384785738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.440359012Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fbbdd412-a2ef-4b2d-9ed5-d16dfb5d3719","remote_addr":"[fd00:1122:3344:101::c]:62842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91515,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:39.457309376Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4dc80f1f-7871-4125-b347-77d062945d5e","remote_addr":"[fd00:1122:3344:101::c]:62842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.408635726Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44157"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.409263671Z","hostname":"buskin","pid":1562,"uri":"/vmms/4188b9f4-4179-44e5-8abb-b1456b39732c/state","method":"PUT","req_id":"46b07699-ec4b-437c-9ceb-c775f8a465ce","remote_addr":"[fd00:1122:3344:101::a]:44157","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":598,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-09-04T16:58:41.409389124Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.409411774Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.409433855Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-09-04T16:58:41.409389124Z }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.409448535Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-09-04T16:58:41.409389124Z }, migration_in: None, migration_out: None }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-09-04T16:58:41.450979795Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451011925Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451045906Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-09-04T16:58:41.450979795Z }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451055027Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:630","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451065757Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:41.451164099Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"3524127187694203785"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451318513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:41.451398375Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/46c20aa9-c404-447d-81be-a48fc73c6ec0/debug/bundle/zone\", \"/pool/int/b11bf0e4-5e3b-4abd-af88-22ad6494addc/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/62c25a09-dd5a-42bc-bdc0-88462c2034ea/crypt/debug\", \"/pool/ext/9859c4c0-8dca-4c62-8980-22f58b7298bf/crypt/debug\", \"/pool/ext/e5004d09-cef9-4e6f-b2aa-6d7ade3877f4/crypt/debug\", \"/pool/ext/f73a9d5e-d846-41dd-90e4-dedfd04db60a/crypt/debug\", \"/pool/ext/84deb833-6610-41c6-8b5e-44fa2aeee752/crypt/debug\", \"/pool/ext/41da29c9-1a43-49cc-b472-5dde9cb41a62/crypt/debug\", \"/pool/ext/d2efc072-fbdb-4b59-a671-9eb348ded4ad/crypt/debug\", \"/pool/ext/5f5397b7-7011-4692-88c2-8a70902c2fea/crypt/debug\", \"/pool/ext/a145c7dc-0acd-4e57-96ae-74d5f1432797/crypt/debug\"] }","zone_name":"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:41.451593419Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"d5abb5c3-38c8-4f7b-a191-64252eb83a69","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"18359982385255979933"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:42.070077294Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1137","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c\", bundle_id: 4b67b699-70b6-4bee-88e2-2df51192a78d }, time_created: 2024-09-04T16:58:41.451782844Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_4188b9f4-4179-44e5-8abb-b1456b39732c","v":0,"name":"SledAgent","level":40,"time":"2024-09-04T16:58:42.070159476Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:826"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:42.232716979Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.823828367Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.842523863Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(15180582), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.842598005Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(15180582), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.842972213Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-09-04T16:58:43.842933533Z }, migration_in: None, migration_out: None }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.866022572Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-09-04T16:58:43.842933533Z }, migration_in: None, migration_out: None }","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:43.870061309Z","hostname":"buskin","pid":1562,"instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","propolis_id":"4188b9f4-4179-44e5-8abb-b1456b39732c","instance_id":"a35778a4-f10e-4179-a3f6-ac70bd84565c","component":"InstanceManager","file":"sled-agent/src/instance.rs:763"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.225764651Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64024"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.226716466Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a2e025a8-4504-48bd-95e1-70fb15dd3954","remote_addr":"[fd00:1122:3344:101::a]:64024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":867,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.241677893Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"44228363-104a-4951-92fa-591d7905a7a6","remote_addr":"[fd00:1122:3344:101::a]:64024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.296181021Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.296349215Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"72bdf1bd-1538-4c93-ab25-22cb5d3dd8d1","remote_addr":"[fd00:1122:3344:101::a]:46069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.704088279Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37244"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:44.704318735Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"51968d5b-50b9-464d-8842-611bd3a79b0a","remote_addr":"[fd00:1122:3344:101::a]:37244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":208,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:45.649407691Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:45.649536134Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"2c70647d-3bf3-427d-8517-3f93fef2ccd2","remote_addr":"[fd00:1122:3344:101::a]:42477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-04T16:58:45.657796001Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"813ce270-68ef-49e0-8cd2-bcc9e1f8b184","remote_addr":"[fd00:1122:3344:101::a]:42477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"}