[ Jul 19 18:41:22 Enabled. ] [ Jul 19 18:41:22 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 19 18:41:22 Method "start" exited with status 0. ] [ Jul 19 18:41:22 Rereading configuration. ] [ Jul 19 18:41:22 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.267349536Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:110"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.267502589Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.319456159Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.319504029Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.435322875Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.444025631Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:22.451997505Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.055295122Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196589585Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196633916Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196686137Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196773318Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196809938Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:718"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.196830249Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.199829035Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:23.199836815Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:767"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.199856036Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.199860396Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:826"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.199876936Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.199922417Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.200027569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.200060479Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.200076739Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.211769222Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.211812102Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"92e7db57-0bba-4f49-937c-e777282668a4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.30977306Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512715894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512758675Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512763855Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512776525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512785375Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512808766Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.512813896Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.52141268Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.52144331Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.60736479Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812197984Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812240695Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812270055Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812278325Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812283925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812290305Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812326656Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812333316Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812345886Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812349856Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812353686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812367766Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812382847Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812397587Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812412417Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812416677Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812420097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812426658Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812464348Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812486688Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812492239Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812517329Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812521539Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812524929Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812528759Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812531729Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81255126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812556969Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81256046Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81256345Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.8125728Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81258534Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81260937Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81261434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812617621Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.81262295Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812626441Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812629511Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812649941Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812661511Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812666421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812670811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812674581Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812680991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812685671Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812706292Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812722192Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812727092Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812730472Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812734892Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812738372Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812742372Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812765953Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812769673Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812772803Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812781713Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812788893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812792483Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812818044Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812822884Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812826114Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812877834Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812904665Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812909505Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812918655Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812927515Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812934945Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812964316Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.812968446Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.823979058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:722"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.824034008Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999937979s","next_cleanup":"Instant { tv_sec: 3264, tv_nsec: 216886528 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.82410054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.82413992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.824198941Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.824208531Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827490652Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827566483Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827790387Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827798917Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827818558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827870118Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827882008Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827887888Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827909959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827920599Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827927799Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827932709Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827936619Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.827940459Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.82795834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.8279806Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828320365Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828492918Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828514038Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828569029Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828581589Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828776813Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.828926695Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.82924545Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 447ca897-407e-4dad-a0e2-4f6036da9497, rack_id: 054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829359962Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:50137","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 447ca897-407e-4dad-a0e2-4f6036da9497, rack_id: 054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829510294Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829517734Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829550154Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829561855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829565355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:572"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829568595Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829576395Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829581685Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829590765Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829598855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:561"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829602215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:589"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829605405Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:393"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829633696Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.829637966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:479","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.83119906Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3815"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.861438882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.882117094Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.898799584Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.905974306Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.906022157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.906034497Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:378"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.906080848Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 65.434574ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.909467641Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9f3b726a-7afa-4937-b237-587da20657c4\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9f3b726a-7afa-4937-b237-587da20657c4\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_92e7db57-0bba-4f49-937c-e777282668a4/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.918320959Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_92e7db57-0bba-4f49-937c-e777282668a4/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.940303001Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_92e7db57-0bba-4f49-937c-e777282668a4/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.940347392Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.945632484Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:393"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.945648965Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_92e7db57-0bba-4f49-937c-e777282668a4/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.945655495Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.976444244Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 85.710857ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:23.977150175Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7b4d4ba-ac23-4c25-b7e1-63e0f1e3c013\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7b4d4ba-ac23-4c25-b7e1-63e0f1e3c013\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.017979422Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.064343535Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 107.53591ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.064977725Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"566db06b-4b95-4ddc-93a7-488e1d868592\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"566db06b-4b95-4ddc-93a7-488e1d868592\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.17306696Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 243.18604ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.173673679Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a154d6f-e792-4a6d-848f-3599a67f619e\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a154d6f-e792-4a6d-848f-3599a67f619e\" }"} {"msg":"Mounting oxi_92e7db57-0bba-4f49-937c-e777282668a4/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.236821214Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.249497022Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.249561723Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.249577313Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.249600903Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.418607029Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 257.265981ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.421013316Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6c1c6dba-63b6-4fd6-818b-d0ebf66be810\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6c1c6dba-63b6-4fd6-818b-d0ebf66be810\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.678778615Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 227.844849ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.679535697Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"787fedfc-90c7-4e3b-852b-c3a5887eb21c\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"787fedfc-90c7-4e3b-852b-c3a5887eb21c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.908609459Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 155.544617ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.909211179Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18af1248-cd0e-4e69-b98d-af5d50312fd3\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:41:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"18af1248-cd0e-4e69-b98d-af5d50312fd3\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.974595018Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.975223367Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.975259598Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:24.975317859Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:35720","producer_id":"447ca897-407e-4dad-a0e2-4f6036da9497"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:24.976574998Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"133.66774ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.021902095Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.021945246Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.022041327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:905","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.022050387Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.02286095Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.022894671Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.022926121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.022949572Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.023219376Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.023406919Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.023437919Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:50137","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.023451989Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.023458879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.02351017Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.02351753Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.207860265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.207919085Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.207973466Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.207978906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.207987137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.208038247Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:25.209008013Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"724.921455ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:25.212566638Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.240119908Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51793"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.240323871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.240458693Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"2076c1f4-7957-4cad-840b-9db1afd899d3","remote_addr":"[fd00:1122:3344:101::1]:51793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":211,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.26976007Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:58436"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.269972743Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"f3bbb839-9950-46cb-b368-0a836f2e05ee","remote_addr":"[fd00:1122:3344:101::1]:58436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.271564748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.271595769Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.271929304Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.272162417Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 567ac47a-d11b-4885-aa37-1cccc1bc2b9d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 9e607437-7489-43a3-8665-a0998816c2da (zpool) }, OmicronPhysicalDiskConfig { id: 86d44b18-8e27-48e8-a454-0d75cb152924, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool) }, OmicronPhysicalDiskConfig { id: aaaf88e5-248a-40ef-9ca8-daffe37e12c5, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool) }, OmicronPhysicalDiskConfig { id: d6fbd8a1-4f59-4b1f-a3d2-30dbc79bce08, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool) }, OmicronPhysicalDiskConfig { id: 65cace0d-8eed-40c9-87ea-79f398426bc6, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool) }, OmicronPhysicalDiskConfig { id: c2285253-bba5-48bb-b7bd-ae3ca4775206, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool) }, OmicronPhysicalDiskConfig { id: 68000882-299b-45e4-8a0b-9f09e9b1f5f4, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool) }, OmicronPhysicalDiskConfig { id: 6407a0ef-8596-481e-898c-0e2437e6766b, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool) }, OmicronPhysicalDiskConfig { id: 2f64fd20-4791-4a75-8ea5-707035293ff6, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303297823Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303396624Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56343"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303658078Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:825"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 567ac47a-d11b-4885-aa37-1cccc1bc2b9d, pool_id: 9e607437-7489-43a3-8665-a0998816c2da (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 86d44b18-8e27-48e8-a454-0d75cb152924, pool_id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: aaaf88e5-248a-40ef-9ca8-daffe37e12c5, pool_id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: d6fbd8a1-4f59-4b1f-a3d2-30dbc79bce08, pool_id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 65cace0d-8eed-40c9-87ea-79f398426bc6, pool_id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c2285253-bba5-48bb-b7bd-ae3ca4775206, pool_id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 68000882-299b-45e4-8a0b-9f09e9b1f5f4, pool_id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6407a0ef-8596-481e-898c-0e2437e6766b, pool_id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 2f64fd20-4791-4a75-8ea5-707035293ff6, pool_id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303709519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303716019Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303721739Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.303729119Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:643"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.30376946Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.3037745Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.30377961Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.30378355Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.409850524Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:25.409882225Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"9e607437-7489-43a3-8665-a0998816c2da (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:26.04324183Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"682.634722ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:26.043429003Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.43188072Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.624806668Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.624874969Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.624888219Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.62490643Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.6249164Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.625085152Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:26.625090432Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:26.831178646Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.524075195s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:27.026920377Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:27.044661744Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:27.218045747Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:27.405337908Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9e607437-7489-43a3-8665-a0998816c2da/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:27.84415185Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 54.350247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:28.038318307Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:28.048813251Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 142.245326ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:28.592727632Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:29.026574556Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 225.970655ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:29.026818569Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:29.223190971Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 358.914441ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:29.768717928Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:29.778418139Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.994425387s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 507.678934ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:30.323416897Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:30.324674856Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_9e607437-7489-43a3-8665-a0998816c2da","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:30.578147689Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 744.454594ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:31.108931874Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.109180148Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.109191418Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.109241589Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.109250769Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.109256569Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:31.340760779Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:31.34081082Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.348376388Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:31.348385808Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool)"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 830.111076ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:32.547150179Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:32.54978936Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:32.830125151Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076499412Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076560333Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076568293Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076575373Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076582213Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076675575Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:33.076692515Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:33.567543759Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:33.567908735Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:34.58481585Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:34.592347628Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9792c5a6-e0f4-4250-91d8-71386ac84474/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.0863216Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:35.329538772Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.544529164Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:35.586986396Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_9792c5a6-e0f4-4250-91d8-71386ac84474","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.663965376Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.672691423Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.672755914Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.672764034Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.672770324Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.672777084Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.682442085Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.682482455Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.772063972Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.779956285Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780000556Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780011506Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780028976Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780047727Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780206539Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.780222739Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:35.783914857Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.472064762s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.816737159Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_61f4fa9a-1baa-4a1b-8076-77046f3a1be7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.824156024Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_61f4fa9a-1baa-4a1b-8076-77046f3a1be7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.948007996Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.955823958Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.955887388Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.955905269Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.955914709Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.955930179Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.965915115Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:35.965951145Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.061370712Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069390657Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069443258Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069453878Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069468089Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069482349Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069587961Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.069604201Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.105964068Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3481a5aa-d464-43f4-8c3b-e7c18df1676b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.114911887Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_3481a5aa-d464-43f4-8c3b-e7c18df1676b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.234875998Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.243472842Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.243517733Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.243539313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.243549523Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.243556393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.253349346Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.253384777Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.34790784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.3568586Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.356895931Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.356905171Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.356972022Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.356984472Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.357134384Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.357156284Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.3927835Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.40178882Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.521885423Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.529166307Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.529219127Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.529225868Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.529231678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.529236578Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.538905558Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.538940689Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.636033413Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644599037Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644651418Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644663828Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644671638Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644684048Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.644776239Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.64478864Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:36.64869539Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.68139613Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.689232973Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.809768332Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.817795587Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.817833698Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.817840938Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.817866848Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.817872999Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.828000306Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.828028277Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool)"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.921615866Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2256"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.934336524Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943303284Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943331525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943336495Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943359445Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943367175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943515527Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.943529838Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.982257371Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2db534d5-2c49-4cc7-bcdd-c78bfddad2ae/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:36.990327557Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:37.11112827Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_2db534d5-2c49-4cc7-bcdd-c78bfddad2ae","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.112525992Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.120277863Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.120329744Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.120336364Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.120342264Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.120347334Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.131094562Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.131131582Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.22910973Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237324688Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237359879Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237367799Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237389169Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237401459Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237546751Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.237556392Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.275432762Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_56328015-6e5b-46f4-8b73-bfdc206d32b1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.283587469Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_56328015-6e5b-46f4-8b73-bfdc206d32b1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.406596187Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.414604822Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.414657943Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.414668403Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.414684794Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.414692094Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.42408892Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.424134581Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.517166042Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525002914Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525058165Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525073485Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525084935Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525095605Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525300678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.525313789Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.562037411Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_59a6abe1-72b0-47e0-8dd6-231a2ecca4d7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.571096543Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:37.65177337Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_59a6abe1-72b0-47e0-8dd6-231a2ecca4d7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.699626337Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707644952Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707706633Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707773054Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707901866Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 92e7db57-0bba-4f49-937c-e777282668a4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b9419da7-be18-4d81-955a-2078bb5c8da4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707982157Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.707991227Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708087028Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708323442Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:829"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708343583Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708352803Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:855","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708359443Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708364973Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:865"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708385883Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708394543Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:871"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708449214Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:822","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708457694Z","hostname":"buskin","pid":1562,"sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.708556146Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"e3d35d1f-ae72-45df-a4fb-271866a821bc","remote_addr":"[fd00:1122:3344:101::1]:56343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12405012,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.739134063Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.739218944Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47249"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.739439368Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.739448328Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.788650905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.788696956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.838244868Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.838293169Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.887384744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.887430675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.887445485Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.887496226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.930769881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.930831462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.930838672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.982480836Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.986028672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:37.989631288Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:38.002173303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:38.007012539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:38.011209374Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:41:38.654557436Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Profile for oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:39.497749233Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:39.497829834Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:39.497981926Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:39.981271731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:40.458751916Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:40.458835008Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:40.458844858Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 30.152852ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.46732903Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:40.467362361Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:40.471545746Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 116.386576ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.50640957Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 27.205296ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.590043144Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 63.433701ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.602349196Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 93.697282ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.625344594Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 229.829155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.632565597Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 146.731311ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.80205603Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 104.084791ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.909866941Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 497.182447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.926761914Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 236.707857ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:40.977159469Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 515.866788ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:41.044053282Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 409.011293ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:41.225076035Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 677.247613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:41.43316076Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 882.085496ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:41.568454309Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 1.019119933s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:41.644691608Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 1.390491594s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:42.120685829Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 1.23781861s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:42.460551839Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 690.164495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:42.673772864Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:42.975890514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2273"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:42.975911414Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:578"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 840.762375ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:43.372908464Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 747.068698ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:43.520700098Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe99:d817","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:43.522471336Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2280"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:43.528362538Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2341"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:43.647055829Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2526"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 1.400909292s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:43.706926702Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:43.889518399Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2690"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:44.08584216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2405"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 1.011970752s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:44.223845532Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1\") failed: Property not found. retry in 1.096909041s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:44.277865824Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:44.388576021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2440"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:44.602674469Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2740"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:44.784748688Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2767"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.006450534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2634"} {"msg":"wait for service Some(\"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e\") failed: Property not found. retry in 991.939059ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:45.117066889Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.181761808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2434"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.19217394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3839","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.192209081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3855"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.192216541Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:561"} {"msg":"wait for service Some(\"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7\") failed: Property not found. retry in 956.265373ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:45.248863614Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.750690279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3863"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.75720366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:45.91620554Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3981"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.017146103Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.038913112Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4009"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.139642513Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4036"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.225879957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_d888dc5c-8f48-4f16-ba5f-c2760b3610b1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.225923348Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_bfbfbed2-0c7b-46ae-9a85-844e2c2af98e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.225928118Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_5ac60829-580a-40c1-8bb7-8a2b487569a7"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.225932478Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.226240453Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.226537068Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"133169e6-bcf5-4a62-9caa-f73910505567","remote_addr":"[fd00:1122:3344:101::1]:47249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8487248,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.228189613Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.262063672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.299442015Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.331118978Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.33119815Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.331666127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.331683287Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"failed to write DNS configuration (will retry in 145.628834ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:46.363607335Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413229049Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.41331554Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.41332613Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.41333386Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.41334096Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.41334654Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413355361Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413362541Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413393951Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413400971Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413407941Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413415162Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413422222Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413428922Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413437552Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413444792Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413453482Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413460842Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413465742Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413470742Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413480373Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413485773Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.413490783Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.528798491Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.580653859Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.627157694Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.627168675Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.659186064Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.661059633Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.661064603Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.689850552Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.689953024Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48066"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.690165617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.690175377Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.708100167Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.708165157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.725937665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.725988995Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.743522459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.74357826Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.743767133Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.743922855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.754896276Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.787249631Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.787989042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.787999873Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.817077886Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 134, 120]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.817090896Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.817471392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.817485982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.817780617Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.818158303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.841806631Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:46.859888163Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:47.468776447Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:47.468914139Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 47.49619ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:47.603155242Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 124.226047ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:47.658316422Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 187.547153ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:47.814730601Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 323.070702ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:48.012548885Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 703.817309ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:48.343293342Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 512.324321ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:49.057679121Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 1.421324731s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:49.579417986Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:49.683722592Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:49.683867395Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e2c96561-4301-4782-a104-19178fb96169","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60341","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 1.164332301s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:51.011379083Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:51.259776566Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"no record found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"23.221330483s"} {"msg":"wait for service Some(\"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50\") failed: Property not found. retry in 787.191956ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:52.185765474Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:52.991498527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:52.991603719Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:52.991969334Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:52.992280819Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c8a8dce0-7404-4e97-a2ce-0749b895a523","remote_addr":"[fd00:1122:3344:101::1]:48066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6302273,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:52.995132154Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.025277354Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.025481577Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61820"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.034748161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.034907674Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"f150bbd9-91ff-456e-a93d-4e041051c09a","remote_addr":"[fd00:1122:3344:101::1]:61820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9292,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:53.036471798Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.124828216Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:53.124965658Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46244"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.133111279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.133355013Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ad1f36a7-bdf1-4e6c-af40-4db7d4e34b8d","remote_addr":"[fd00:1122:3344:101::1]:46244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2008248,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:55.136421051Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.28706558Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.287192622Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53285"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.295931318Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.2960768Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"f7b11a9d-f714-450c-b0cf-16f76b21d8a5","remote_addr":"[fd00:1122:3344:101::1]:53285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8762,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:55.299007906Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.464571307Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:55.4647384Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49338"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_8bffd25b-2901-4312-b7a6-dab1af509a50': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.473368432Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.473543585Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"dc1ba3cb-9d3f-450d-a4a5-bac99063d8be","remote_addr":"[fd00:1122:3344:101::1]:49338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008685,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:59.476682924Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.667116653Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.667248366Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51447"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.676080983Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"023f0ac3-54fa-4a16-87a1-654389d6525f","remote_addr":"[fd00:1122:3344:101::1]:51447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8680,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.678123235Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:59.678129445Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.872664389Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.872802221Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.880143745Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d9c31d63-70d2-4624-a36a-90e587cac084","remote_addr":"[fd00:1122:3344:101::1]:47179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7213,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:41:59.882773086Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:41:59.882779226Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.107745634Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.107862796Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.116852896Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"f9c470b7-67e5-4ebb-b1c9-774569d4320b","remote_addr":"[fd00:1122:3344:101::1]:59578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8874,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.120040985Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:00.120048886Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.316011481Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.316092642Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41892"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.323937865Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1b5b2732-d600-42c8-ba6c-528b9a54d1fe","remote_addr":"[fd00:1122:3344:101::1]:41892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7767,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.325867555Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:00.325871595Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.741424234Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.741573957Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60511"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.750426115Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"612451ff-9ce1-4580-9ea7-91dc2242705f","remote_addr":"[fd00:1122:3344:101::1]:60511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8801,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:00.75271536Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:00.752723771Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.201082301Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.201208533Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.209135196Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"f0aedd4c-146a-4b4e-892e-1818682ab608","remote_addr":"[fd00:1122:3344:101::1]:55487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7802,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.211886749Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:01.211893139Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.552369908Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.55248744Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.559785204Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"029d5dc4-6008-4620-944c-4e7cc14b2a2c","remote_addr":"[fd00:1122:3344:101::1]:34142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7207,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.562065259Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:01.56207733Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.99502596Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:01.995187722Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.005050036Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"82b7edeb-2a02-4c02-96ff-a291f58ec73a","remote_addr":"[fd00:1122:3344:101::1]:44911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9800,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.006922965Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:02.006926975Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.81274946Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.812874622Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.82106156Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"c11a7a29-ad34-4195-89bd-e2248c367f6a","remote_addr":"[fd00:1122:3344:101::1]:57748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8059,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:02.823332325Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:02.823340616Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.175597668Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.175762431Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63222"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.195763423Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"2f82e6fb-8ac9-41b9-a0a4-60ceb218d8ab","remote_addr":"[fd00:1122:3344:101::1]:63222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":19940,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 759117325, ip_addr: 45.63.54.13, stratum: 3, ref_time: 1721414522.8769734, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.199492081Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.199499451Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.229009731Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.229143583Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:64241"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.229371147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.229376747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.248465364Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.248509965Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.267786096Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.267839047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.284843392Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.284870462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.335760935Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.335794866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.386575068Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.386610588Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.43610087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.436135691Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.489702166Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.489735367Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.541857539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.54189524Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.542146904Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.542265395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574494088Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574535429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574544789Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574552249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574561419Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.574671051Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"54049cfc-b9a5-4406-90ed-a34c7ad6d6d7","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\", TimeNotSynchronized), (\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\", TimeNotSynchronized), (\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\", TimeNotSynchronized), (\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\", TimeNotSynchronized), (\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\", TimeNotSynchronized)]","latency_us":345428,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.574908644Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"140.440008ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"54049cfc-b9a5-4406-90ed-a34c7ad6d6d7\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"54049cfc-b9a5-4406-90ed-a34c7ad6d6d7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.716090755Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.716553743Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.716562852Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.735287355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.735316985Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.753271995Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.753314576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.770854229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.77089548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.788983552Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.789015392Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.806639597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.806668408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.824202401Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.824241832Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.841815916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.841856366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.859071955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.859114575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.859331179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.859463441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891302077Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891351948Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891360868Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891366278Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891371578Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.891429239Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9cc8630c-78aa-4493-97e7-e95812fd1472","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\", TimeNotSynchronized), (\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\", TimeNotSynchronized), (\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\", TimeNotSynchronized), (\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\", TimeNotSynchronized), (\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\", TimeNotSynchronized)]","latency_us":175120,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:03.891586262Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"79.49364ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9cc8630c-78aa-4493-97e7-e95812fd1472\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9cc8630c-78aa-4493-97e7-e95812fd1472\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.972739137Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.973090992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.973103973Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.991786304Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:03.991828595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.011478011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.011510852Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.029669565Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.029727576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.048311525Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.048346156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.066846765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.066885255Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.084419408Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.084453219Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.103351204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.103388524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.121024329Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.12107194Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.121311474Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.121419696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.153855911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.153918602Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.153929342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.153937243Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.153945072Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.154042784Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"48f40175-d863-481e-bb04-6385a7889d23","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\", TimeNotSynchronized), (\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\", TimeNotSynchronized), (\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\", TimeNotSynchronized), (\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\", TimeNotSynchronized), (\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\", TimeNotSynchronized)]","latency_us":181147,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.154249517Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"168.195256ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"48f40175-d863-481e-bb04-6385a7889d23\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"48f40175-d863-481e-bb04-6385a7889d23\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.323644889Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.324001274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.324017655Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.34233932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.342372201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.360466803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.360517893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.37821426Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.37825055Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.396107608Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.396146779Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.414410434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.414443734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.431944157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.431973798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.449880647Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.449914558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.468578989Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.46863785Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.468866383Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.469018376Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501491922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501539523Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501547523Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501555473Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501563093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.501668905Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1193f0c8-bd8a-4dff-9f8e-d9e55d2f8898","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\", TimeNotSynchronized), (\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\", TimeNotSynchronized), (\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\", TimeNotSynchronized), (\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\", TimeNotSynchronized), (\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\", TimeNotSynchronized)]","latency_us":177867,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.501863068Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"257.14645ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1193f0c8-bd8a-4dff-9f8e-d9e55d2f8898\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1193f0c8-bd8a-4dff-9f8e-d9e55d2f8898\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.760258426Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.761634957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.761641578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.780159486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.780215347Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.798121816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.798163027Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.815808612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.815849453Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.833928215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.833966675Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.85225858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.852312201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.870539185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.870582066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.888922152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.888953963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.90672436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.906781631Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.907017254Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.907156247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939209496Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939244627Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939250987Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939266007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939273327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:04.939348929Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"653b43f8-8c1f-4aaf-ba9b-7f5cda1f099e","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\", TimeNotSynchronized), (\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\", TimeNotSynchronized), (\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\", TimeNotSynchronized), (\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\", TimeNotSynchronized), (\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\", TimeNotSynchronized)]","latency_us":178923,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:04.939493041Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"248.340402ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"653b43f8-8c1f-4aaf-ba9b-7f5cda1f099e\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"653b43f8-8c1f-4aaf-ba9b-7f5cda1f099e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.188875199Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.189227265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.189235925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.207710393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.207759854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.225856426Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.225886916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.008397001Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.008429192Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.026689836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.026726857Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.045403038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.045434278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.064880122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.064928302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.083695265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.083737786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.102469598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.102509988Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.102720672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.102833273Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.185814087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.185858458Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.185866168Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.185870948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.185877128Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.279995536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.284340884Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.288730622Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.293115091Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.297672171Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.302494927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.307105119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.311813512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.316534766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:05.320993215Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.187660462Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.187972847Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.193858169Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.194045662Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.2959056Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.296219155Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.303467958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.30358598Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.312078582Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:08.312194354Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 31.613416ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.331613547Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 62.842985ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.338307431Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 75.40261ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.371767383Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 122.053741ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.468072724Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 166.942345ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.498033201Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 31.334888ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.618066623Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 60.363627ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.755456755Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 38.198806ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.846771979Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 345.659259ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:08.962598955Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 261.711751ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.009610208Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 148.964846ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.034827621Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 114.776129ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.071635155Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 100.963191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.108220076Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 180.813833ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.210897277Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 119.492635ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.265911604Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 107.761168ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.317162323Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 204.048172ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.338052569Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 813.591858ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.410467198Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 371.420728ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.482943728Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 544.339917ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.662504518Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 408.086441ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:09.884210925Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 675.666075ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.089473335Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 597.739947ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.292126885Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 886.681683ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.708850222Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 767.626107ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.795505264Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 1.004991158s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.905813724Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 855.239344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:10.986776846Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 1.1739545s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:11.001261812Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 536.891287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:11.586854442Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 1.359490281s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:11.600469144Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 1.28197542s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:11.854241952Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 1.317703519s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:11.922694359Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.28679612s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:12.134524572Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 1.05385134s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:12.185134931Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 536.113639ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:12.973443172Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 1.251319557s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:13.150459922Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 1.486023842s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:13.253210174Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 1.045916152s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:13.266850247Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 953.929505ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:13.433895281Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 1.430080356s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:13.52108745Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 1.176808854s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:14.324535889Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.365599418s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:14.397193931Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 1.326700011s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:14.410536239Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 1.297493549s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:14.747092706Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 991.615178ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:14.959194444Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31\") failed: Property not found. retry in 1.370844694s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:15.511129929Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af\") failed: Property not found. retry in 1.245566463s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:15.747389743Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 648.833602ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:15.773019313Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 539.181101ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:15.960697889Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402\") failed: Property not found. retry in 1.237489568s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:16.05436566Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 756.708349ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:16.433231127Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2\") failed: Property not found. retry in 698.804994ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:16.513352306Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.254396256s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:17.201829311Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 711.398565ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:18.467728869Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 716.377613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:19.190024771Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:19.190169643Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:19.190315036Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6850a320-ce79-4816-b361-a1d89fd5e14f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56345","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 627.220215ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:19.918839655Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 990.466483ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:21.09095115Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.467784137s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:22.094327075Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.497537242s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:23.571960964Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.400595798s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:25.079633492Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8\") failed: Property not found. retry in 1.495482531s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:26.489967091Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007482293Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_70f181a1-2827-4ca2-aa37-744b188fafb8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007540274Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_c1bda75f-7bfe-4403-94af-34867f337f31"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007545654Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_808f488e-12f3-4bfc-aef1-cad905bec1af"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007549964Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_8091f6e2-58ac-4ea2-be97-ffe07a6d4402"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007555034Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_61cb6caf-7127-44ff-b97d-23c7d6df94f2"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.007597945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.024837704Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.046665114Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"02cfab72-bcfb-424b-b40a-872dea735ae1","remote_addr":"[fd00:1122:3344:101::1]:64241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":23093599,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.080316618Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39427"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:28.125012826Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3403"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:29.154297304Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3421"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.18532378Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.185457302Z","hostname":"buskin","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"a78df896-72bf-48d1-b3d0-c1bbed791d75","remote_addr":"[fd00:1122:3344:101::1]:39427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":22105013,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.185815547Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.18726968Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"20b9d2a9-7e7c-458e-839c-284bc610fe87","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42611","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1273,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:50.190591772Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"308.521259ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.22058745Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.220744122Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50948"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.221134828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.221140818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.245148203Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.245200384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.265567631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.265601672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.285241638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.285274868Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.304235254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.304267814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.321713786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.321746617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.340698783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.340736703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.359012688Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.359046279Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.377318054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.377350574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.517228745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.517273855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.652002246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.652049447Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.795349241Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.795392922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.907859316Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:50.907899346Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.025155105Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.025197145Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.169554996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.169593847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.27366678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.27370926Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.399573143Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.399610753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.554177662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.554221133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.670492196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.670534527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.795545946Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.795587137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.986161128Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.986225319Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.986448633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:51.986597615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.135981714Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136029715Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136037935Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136042125Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136048825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136054095Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136059546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136064936Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136071586Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136077336Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136082756Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136087886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136103306Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136108616Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136112046Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136117387Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136120887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136124556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136128277Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.136218528Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4b4b552e-9e72-4063-baf6-ee7e0c74e621","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized)]","latency_us":1915320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.136504642Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"84.849256ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b4b552e-9e72-4063-baf6-ee7e0c74e621\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4b4b552e-9e72-4063-baf6-ee7e0c74e621\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.222717057Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.223090103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.223096913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.243669224Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.243703684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.2633481Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.263384321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.282438518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.282472709Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.30308908Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.303123851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.321364605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.321398445Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.339859563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.339901104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.35955127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.359583211Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.378450485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.378500396Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.39736059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.39739039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.41594936Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.41598082Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.434925845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.434958586Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.453412554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.453437954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.473257853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.473295713Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.490808597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.490833307Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.508870078Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.508898099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.527089672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.527119863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.545464579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.545491869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.563359318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.563390108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.582904153Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.582935523Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.604695962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.604725253Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.604958546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.605092839Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.703835258Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.7039162Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.7039266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70393521Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70394223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70395073Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70395773Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70396561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70397221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.703978681Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.70398341Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.703989961Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.703996561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704004901Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704012071Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704018391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704032411Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704039131Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704045131Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.704179864Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"af436ef6-0728-47b7-997b-c879956e0a19","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized)]","latency_us":481275,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:52.704431717Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"147.103001ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"af436ef6-0728-47b7-997b-c879956e0a19\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"af436ef6-0728-47b7-997b-c879956e0a19\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.852642328Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.852962133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.852970084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.875393483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.875444684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.896498472Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.896551013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.914436162Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.914485943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.931909255Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.931961035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.951869216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.951913707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.970850372Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.970901143Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.989598704Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:52.989641955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.00796195Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.008023721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.030673364Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.030722885Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.050354151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.050403052Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.07140986Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.07145624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.091018676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.091066546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.109792038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.109846739Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.127626806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.127672767Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.146566631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.146615092Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.166013475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.166078556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.186817559Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.18686414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.205705304Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.205748854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.223670414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.223716694Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.240611858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.240661369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.240871732Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.240999304Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335876494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335927814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335935814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335942384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335947655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335972305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335979135Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335984355Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335990325Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.335995445Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336001655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336007256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336013376Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336017426Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336023576Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336028706Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336034796Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336041036Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336045216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.336180098Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4fdd0c4e-0557-4a96-bec8-0c17df9e59f7","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized)]","latency_us":483397,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.336427212Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"157.565874ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fdd0c4e-0557-4a96-bec8-0c17df9e59f7\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4fdd0c4e-0557-4a96-bec8-0c17df9e59f7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.494976774Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.49541354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.495430971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.517994702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.518029833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.538417741Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.538457952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.557295685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.557332236Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.575507579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.57554602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.594966833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.595000373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.61274414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.61278254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.633388972Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.633423802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.65252146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.652555931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.671282153Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.671329513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.689555668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.689590288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.709088382Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.709122863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.728292992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.728328152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.747696114Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.747740055Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.76663941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.76668941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.786296716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.786342797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.806061094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.806094935Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.825801402Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.825831973Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.844572455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.844603605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.862971772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.863004132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.88082612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.880878251Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.881155495Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.881312498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975649698Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975709119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97571587Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97572346Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97572939Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97573536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97574266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.97574868Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975756Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975784351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975790561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975796341Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975803401Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975807701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975813171Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975818921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975824381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975830481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.975836132Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:53.975986804Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"78c99ce6-0d1c-4e1d-a0c4-a6dccda15ca7","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized)]","latency_us":480855,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:53.976218257Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"237.992736ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78c99ce6-0d1c-4e1d-a0c4-a6dccda15ca7\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78c99ce6-0d1c-4e1d-a0c4-a6dccda15ca7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.215724142Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.216107858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.216118948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.240098852Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.240151303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.263009629Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.26307803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.284741018Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.284798419Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.307169908Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.307219419Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.327868501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.327917921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.348590094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.348653355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.368320391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.368364132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.387550881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.389033574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.409371541Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.409419412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.428682402Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.428725422Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.449339554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.449382834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.4683239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.468368891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.489461869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.48950785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.509503102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.509542193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.529162198Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.529222979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.54914648Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.549188501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.567833221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.567879182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.586679345Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.586728956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.606610906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.606663087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.62614202Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.626203101Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.626416375Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.626566637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720140266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720199797Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720209217Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720217558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720224087Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720231718Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720238198Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720245558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720251858Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720259268Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720263758Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720270058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720274668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720297699Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720304119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720310309Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720316409Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720320779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720326509Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.72037742Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b06fe68b-e99f-4e37-8743-9af176460922","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized)]","latency_us":504494,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:54.720635584Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"152.169336ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b06fe68b-e99f-4e37-8743-9af176460922\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b06fe68b-e99f-4e37-8743-9af176460922\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.873843193Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.874637185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.874644925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.8967551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.896807741Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.917457133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.917501744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.937573737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.937621067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.958253669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.95829729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.976700077Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.976745378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.996870361Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:54.996915952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.017553264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.017617585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.037482885Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.037524515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.057268613Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.057308424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.076733617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.076771817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.096580416Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.096630757Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.115283087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.115319308Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.134858603Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.134903753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.153958881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.153997191Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.172699013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.172739883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.192021294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.192065545Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.212662176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.212702387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.231681793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.231720633Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.251043324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.251086545Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.269726036Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.269775256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.26997918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.270094581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367705894Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367777455Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367786015Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367791375Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367798085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367803935Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367809505Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367816645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367821915Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367827436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367834236Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367840846Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367847666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367851916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367856976Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367862576Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367884266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367892617Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.367897917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.368033199Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7f6ef924-9c86-4565-be2a-750ea15c5d96","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized)]","latency_us":494043,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:55.368271792Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"367.447275ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f6ef924-9c86-4565-be2a-750ea15c5d96\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7f6ef924-9c86-4565-be2a-750ea15c5d96\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.737805434Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.73819188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.73819915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.76126637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.7613191Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.781749509Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.78179928Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.802174997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.802224678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.820350491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.820395472Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.839048463Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.839093103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.858689249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.858726439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.876861842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.876909343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.895190768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.895256699Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.914891095Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.914929216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.933268482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.933317293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.950987248Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.951026958Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.968241097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.968291238Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.986783766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:55.986828797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.006476043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.006516994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.026619507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.026665998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.045855157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.045898988Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.064767752Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.064814863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.084933637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.084985797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.10376005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.103805791Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.123319545Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.123370446Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.123597289Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.123743782Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217346361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217388412Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217394792Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217399262Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217403632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217417612Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217423463Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217427603Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217431932Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217437013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217441113Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217444373Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217448663Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217452703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217455973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217460843Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217464863Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217468703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217472853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.217596805Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c13d38a1-d7ca-4002-b92f-f3add83bebeb","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized)]","latency_us":479642,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:56.217823389Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"429.761066ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c13d38a1-d7ca-4002-b92f-f3add83bebeb\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c13d38a1-d7ca-4002-b92f-f3add83bebeb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.64830258Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.648666016Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.648681996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.671552703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.671594823Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.692081703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.692117863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.712836446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.712872797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.731988245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.732030866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.750660366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.750699127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.770713829Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.770749879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.789175737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.789208137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.809378462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.809416862Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.827944321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.827980832Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.846872136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.846908827Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.866767556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.866800927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.884837518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.884871109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.90292598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.902962841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.922206261Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.922252682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.942158022Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.942193043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.96190604Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.96193818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.980851595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.980887476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.999480086Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:56.999526817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.019149913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.019191523Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.037951256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.037988716Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.0382041Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.038348572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133109069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.13316999Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133178361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133185501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133191481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133197351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133203291Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133210501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133216211Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133232251Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133238342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133244032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133251212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133256812Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133263862Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133268092Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133273562Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133280512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133284572Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.133435375Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"427387d1-d6d3-4894-a12b-31aa1df20f48","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized)]","latency_us":484976,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.133670978Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"342.317115ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"427387d1-d6d3-4894-a12b-31aa1df20f48\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"427387d1-d6d3-4894-a12b-31aa1df20f48\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.477018722Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.477397837Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.477410118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.499825567Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.499889048Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.517775087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.517826118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.537036147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.537084878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.554896446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.554947747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.575976245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.576024955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.594992961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.595053662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.613377728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.613424478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.632957473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.633010454Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.653167098Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.653225459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.674002023Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.674054974Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.692596543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.692647784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.711242034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.711288075Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.72833223Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.728376851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.745790722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.745832113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.766708048Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.766762499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.785498541Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.785546632Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.804715271Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.804762112Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.824621562Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.824669932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.843381144Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.843429815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.862775317Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.862839178Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.863053391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.863195803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956109492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956156682Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956164153Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956170683Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956175933Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956180863Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956184823Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956189963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956195013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956198933Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956204813Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956209753Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956215763Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956232114Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956238514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956243544Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956248724Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956254774Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.956259784Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:57.956370746Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1847b399-e227-409b-856e-9a173491ad69","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized)]","latency_us":479213,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:57.95662577Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"485.019008ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1847b399-e227-409b-856e-9a173491ad69\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1847b399-e227-409b-856e-9a173491ad69\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.44215234Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.443034344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.443046344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.464865384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.464904674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.486177176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.486211847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.50568773Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.505720201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.525643721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.525674282Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.544228811Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.544268022Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.563519272Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.563553473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.581704896Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.581745516Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.5999313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.59996076Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.619490375Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.619518605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.637257042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.637280522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.655559197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.655587898Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.674288529Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.67431628Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.692969361Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.693000501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.710649626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.710683937Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.728389023Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.728421013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.745250476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.745277436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.764470945Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.764504146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.781589492Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.781616343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.7993702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.79939948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.819408452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.819451113Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.819669676Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.819788758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.91223586Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912309081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912317511Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912323391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912327311Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912334021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912339391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912344331Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912349411Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912354351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912359311Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912362982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912368362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912372032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912375752Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912379342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912388472Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912393452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912398392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:58.912465993Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"da612ef4-94a2-4ac5-a1e8-9109c8cfdaeb","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized)]","latency_us":470148,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:58.912719897Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"546.389288ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da612ef4-94a2-4ac5-a1e8-9109c8cfdaeb\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da612ef4-94a2-4ac5-a1e8-9109c8cfdaeb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.460435907Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.460904734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.460918014Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.483342794Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.483395185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.504477734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.504518864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.522376973Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.522433574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.540355593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.540403494Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.558588997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.558638338Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.579249749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.57929272Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.598660692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.598705403Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.617974043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.618031024Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.636534803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.636575193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.656536614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.656578705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.675985008Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.676026178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.694683429Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.69472594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.714777233Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.714837994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.733568586Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.733611156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.751528376Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.751582336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.769278152Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.769319603Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.787946394Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.787988434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.807462938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.807504668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.826058798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.826098518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.844076899Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.84412191Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.844344713Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.844467895Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943621821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943679252Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943687762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943695022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943700912Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943726643Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943732333Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943737773Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943741823Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943747203Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943753203Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943757673Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943763133Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943769943Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943773753Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943779033Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943784613Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943791564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.943796624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:42:59.943919706Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0aad83f1-3b2a-415b-bb1e-35871c3757b4","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized)]","latency_us":483228,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:42:59.944157379Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"726.398447ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0aad83f1-3b2a-415b-bb1e-35871c3757b4\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:42:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0aad83f1-3b2a-415b-bb1e-35871c3757b4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.671187505Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.672470435Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.672482845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.693042866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.693080816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.711622005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.711662286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.732340508Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.732378099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.751086551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.751121991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.770356801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.770404302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.788848949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.78888754Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.807373369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.807404909Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.827919929Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.827954229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.846508959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.846540579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.866219006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.866247327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.884587352Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.884616713Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.905141773Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.905176863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.924314322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.924345342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.942655748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.942704899Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.960942923Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.960977184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.980988545Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.981030996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.99925148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:00.999281231Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.02035889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.0203935Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.038497262Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.038535603Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.058188739Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.0582327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.058509384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.058653956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150865904Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150931886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150944536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150952226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150959486Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150966696Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150976086Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150983556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.150992656Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151019867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151028307Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151038097Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151045357Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151050397Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151057657Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151064717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151071738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151079068Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151084418Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:01.151090518Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"19bb8abf-9308-468b-9ec4-1e3bf3027dc6","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized)]","latency_us":479681,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:01.151328972Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"911.507247ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"19bb8abf-9308-468b-9ec4-1e3bf3027dc6\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"19bb8abf-9308-468b-9ec4-1e3bf3027dc6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.064074873Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.064450039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.064458799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.088088807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.088132648Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.109120086Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.109167346Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.129555184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.129603885Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.150544441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.150587592Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.17031362Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.17036225Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.189816714Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.189879045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.208873661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.208911102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.227143636Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.227186927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.246200223Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.246237413Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.265676697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.265710547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.285180101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.285224801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.303809531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.303842482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.322476572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.322514603Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.341387537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.341422968Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.360679028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.360722559Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.379576783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.379620743Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.398717331Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.398758362Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.418344597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.418397038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.437751619Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.43779579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.457638469Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.45769927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.457903133Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.458035366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554773344Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554821035Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554828965Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554834335Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554839555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554845835Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554851085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554855075Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554861265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554867495Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554873565Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554878785Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554883816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554908846Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554914536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554918466Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554923666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554927546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.554932726Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:02.555004487Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b5c7c218-0a7b-4a00-8ef6-19d16c06f6ed","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized)]","latency_us":490787,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:02.55519813Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"681.798061ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5c7c218-0a7b-4a00-8ef6-19d16c06f6ed\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5c7c218-0a7b-4a00-8ef6-19d16c06f6ed\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.23884186Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.239179196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.239187016Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.262004202Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.262042582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.282339289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.282370359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.300250338Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.300290979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.32022803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.32026211Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.338605066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.338639137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.357019113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.357051984Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.375836346Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.375878057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.393699005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.393733356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.412126502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.412159703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.430532649Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.430565429Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.448915075Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.448947086Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.468135405Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.468174956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.486270787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.486299188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.505535398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.505566319Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.525131133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.525164334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.544432625Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.544476785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.56273551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.562772571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.581154707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.581191108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.601990042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.602024343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.620875206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.620909697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.62110536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.621203922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715066425Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715138276Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715145366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715151147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715156516Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715161827Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715167117Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715172287Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715177547Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715183117Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715188337Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715193657Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715200397Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715205577Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715209497Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715214697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715221357Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715239948Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715245838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:03.715348939Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"18096891-c424-4244-bb48-c62f67a68991","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized)]","latency_us":476364,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:03.715584573Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"988.24695ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18096891-c424-4244-bb48-c62f67a68991\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"18096891-c424-4244-bb48-c62f67a68991\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.705311325Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.705672731Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.705687831Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.728396735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.728436136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.748468428Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.748501968Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.766176514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.766206955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.785007088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.785038808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.804043574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.804072915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.821828572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.821858742Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.840489983Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.840522543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.859887465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.859915476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.878018948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.878042758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.896313503Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.896344944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.913845677Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.913880847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.931314599Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.93134247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.94933397Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.949369431Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.969160549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.9691913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.987853641Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:04.987883931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.006887628Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.006916208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.026156808Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.026189498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.045445889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.045475149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.066947544Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.066982194Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.086755293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.086788203Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.086971166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.087084498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180086258Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180140399Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180146589Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180153709Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180157469Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180171249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18017679Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18018225Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18018881Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18019409Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18019958Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18020478Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18021032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18021687Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18022236Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18022787Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18023445Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.18023998Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180246651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:05.180339442Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"41489406-b956-4cb6-9b25-dbcd412ed332","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized)]","latency_us":474869,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.180572826Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.130357976s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41489406-b956-4cb6-9b25-dbcd412ed332\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41489406-b956-4cb6-9b25-dbcd412ed332\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:05.546302728Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"662.28702ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.31279175Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.313121025Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.313129745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.337254881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.337306922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.360876989Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.36092647Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.385523733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.385573814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.407070149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.407127041Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.428664756Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.428711556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.449948917Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.449994718Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.471116598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.471166588Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.492039234Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.492094965Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.513242894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.513299485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.534391124Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.534433895Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.556138053Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.556190944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.576915337Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.576964928Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.598406532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.598455943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.620530177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.620582638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.643301492Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.643349383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.664045806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.664089747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.685611672Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.685659883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.70537127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.705440471Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.726121134Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.726167084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.746691684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.746748025Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.746994109Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.747163392Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.84836785Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848408581Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848415481Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848419111Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848424261Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848428561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848433731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848447241Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848451631Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848456731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848461071Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848466051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848469291Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848473382Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848478222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848482222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848486392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848491322Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848495462Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:06.848579013Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"13d5ba43-7a5e-409b-ac79-8b143822fb6f","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized)]","latency_us":535670,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:06.848800687Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.338941024s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13d5ba43-7a5e-409b-ac79-8b143822fb6f\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13d5ba43-7a5e-409b-ac79-8b143822fb6f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.189711594Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.190548597Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.190560557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.214231727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.214283298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.235879054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.235927365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.255022483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.255071923Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.276478237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.276518548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.295724727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.295764058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.315467065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.315506186Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.334614184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.334665315Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.353399797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.353440357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.376255143Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.376300324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.396487768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.396538209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.416985818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.417027509Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.435979004Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.436024614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.456965591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.457009162Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.478112411Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.478156651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.498948196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.498992216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.518686183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.518727004Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.54093818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.540986211Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.561387209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.56142858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.581818668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.581857918Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.601929151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.601993712Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.602226886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.602359618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705494126Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705559947Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705567927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705572767Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705582938Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705588498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705593008Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705597308Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705600878Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705605628Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705610848Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705628058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705639758Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705649648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705654008Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705658299Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705662039Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705666429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705669979Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:08.7057459Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"08474b25-b305-4954-8502-21456e29343a","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized)]","latency_us":515881,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:08.705958803Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.37993479s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"08474b25-b305-4954-8502-21456e29343a\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"08474b25-b305-4954-8502-21456e29343a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.789856101Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.790172976Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.790181436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.813027983Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.813092894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.834402397Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.836392878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.855867523Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.855921734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.876539426Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.876591197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.897506904Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.897568025Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.918914358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.918977159Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.939942657Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.940000038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.961459464Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.961525835Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.984142148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:11.98421279Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.004829482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.004882883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.026215456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.026290587Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.047928546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.047987476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.06867045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.068728501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.089857651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.089908582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.110573345Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.110636786Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.131138746Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.131201267Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.153188301Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.153239492Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.17552676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.175587991Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.196362386Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.196418956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.217641638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.217710931Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.217912973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.218036885Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320534427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320591408Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320597468Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320602178Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320606558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320610358Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320618549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320623149Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320627369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320631659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320636139Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320641679Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320646339Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320654839Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320668099Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320675779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.32068027Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.32068385Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320687139Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:12.320763301Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"01e9cd97-0235-4bb3-95db-a44226156eb9","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized)]","latency_us":532219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:12.320969024Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.924201833s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"01e9cd97-0235-4bb3-95db-a44226156eb9\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"01e9cd97-0235-4bb3-95db-a44226156eb9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.239262569Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.239675256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.239683926Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.263280965Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.263347816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.283560912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.283613173Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.303732637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.303801418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.32631297Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.326375682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.346470746Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.346522046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.366716692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.366770393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.388387801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.388452412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.408495015Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.408553446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.431258441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.431319112Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.453265006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.453320026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.473526552Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.473577793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.494136334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.494189585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.514772904Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.514834278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.533600802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.533658382Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.554149763Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.554205714Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.574181296Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.574243847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.595174934Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.595235505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.615116356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.615166687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.637265952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.637322063Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.65502944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.655082761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.655281134Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.655437556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749337275Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749410826Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749422206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749447856Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749454196Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749461626Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749474967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749480987Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749488927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749495017Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749500707Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749508137Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749518467Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749525778Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749531598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749538888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749544738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749558208Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749571088Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:15.749659269Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a9afba55-8542-47ea-aef6-2353adc65771","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized)]","latency_us":511607,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:15.749865773Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.380585107s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9afba55-8542-47ea-aef6-2353adc65771\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9afba55-8542-47ea-aef6-2353adc65771\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.124758052Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.125174059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.125183939Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.149061212Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.149119943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.170403976Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.170483067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.192626113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.192681414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.213452549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.215849146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.234311695Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.23437654Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.254915017Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.254969158Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.275153324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.275215785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.29537642Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.295432701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.317189641Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.317252992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.338660646Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.338720357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.359445751Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.359508022Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.379079498Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.379141939Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.399063411Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.399120322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.421754076Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.421817517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.442553661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.442605762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.462620155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.462676145Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.484354134Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.484425205Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.50519744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.505253721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.524830557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.524887088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.544607166Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.544676067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.544896861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.545037843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644649831Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644728762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644736522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644744272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644750402Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644761302Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644768912Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644796543Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644804353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644825063Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644835204Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644842044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644850164Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644858364Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644865534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644871054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644877184Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644881514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.644887324Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:18.644930785Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"32755d86-d6d7-4b6c-8018-07fb08e64dc3","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized)]","latency_us":521416,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:18.645137998Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.911526756s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32755d86-d6d7-4b6c-8018-07fb08e64dc3\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"32755d86-d6d7-4b6c-8018-07fb08e64dc3\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:20.898974846Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:20.899043017Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"57cde200-97cc-4be9-b81f-981502ae7bc0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38899","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.549816081Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.550201187Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.550214927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.571810175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.571858046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.592959776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.593005906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.612492061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.612537822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.63096716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.6310083Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.650694588Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.650736249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.66871626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.668762271Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.689874311Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.689917161Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.709784572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.709825483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.728389783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.728442914Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.746837035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.746873882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.766975456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.767016697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.786531602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.786575213Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.804748817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.804791678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.822537785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.822577936Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.843349731Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.843397901Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.861963062Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.862005652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.882031455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.882070536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.902118889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.90216779Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:21.918451635Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"676.27622ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.920304684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.920316594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.940760294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.940808564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.940996277Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:21.941116109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042745238Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042812059Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042819539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042827179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.04283227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.0428672Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.0428739Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.04288144Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.04288885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.04289474Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042908711Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042914921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042920741Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042927821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042933451Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042940361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042945991Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042951501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.042958701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:22.043052733Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"92b6457d-0ad7-4f27-802a-1413734ac5c4","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized)]","latency_us":494404,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:22.043294517Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"4.543016787s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92b6457d-0ad7-4f27-802a-1413734ac5c4\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92b6457d-0ad7-4f27-802a-1413734ac5c4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.574646011Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.574999586Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.575023817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.597484518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.597536949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.617651703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.617718954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.639562295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.639611727Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.65837245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.658427141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.677935876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.677999747Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.699356791Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.699406882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.718345247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.718396248Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.738384241Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.738433882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.757053493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.757100114Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.77795979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.77800478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.798183676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.798230447Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.818999151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.819048979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.839817247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.839873888Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.85919369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.859261261Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.879401056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.879448996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.899597091Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.899649282Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.921284061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.921341342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.944108898Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.944164668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.964529927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.964588638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.985258461Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.985324242Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.985531225Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:26.985676137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091044454Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091102035Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091108655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091114305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091118755Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091132355Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091136816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091142345Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091146606Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091165756Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091170456Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091175856Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091181296Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091185606Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091190156Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091194416Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091199317Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091203606Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091208786Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:27.091311147Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cfa552f6-6a56-4243-ba9e-cd3b04afebb7","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized)]","latency_us":517913,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:27.091525582Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.17480636s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cfa552f6-6a56-4243-ba9e-cd3b04afebb7\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cfa552f6-6a56-4243-ba9e-cd3b04afebb7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.250892349Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.251256374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.251264365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.274857863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.274903384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.297277894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.297325815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.31816548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.318211301Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.34119476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.341241341Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.359940614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.359994265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.381613332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.381671183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.404122344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.404174905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.424047126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.424089467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.445253647Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.445299828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.466648852Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.466691683Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.490480294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.490523815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.51129357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.511339851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.532321649Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.532371589Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.554902842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.554949233Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.576359927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.576405058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.598764888Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.598818439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.619750426Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.619794596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.64050921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.640550571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.660787847Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.660829318Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.682986864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.683053605Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.683219738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.68334479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784570803Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784649034Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784656814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784664174Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784670664Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784676764Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784689494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784694074Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784700615Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784706685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784719155Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784725575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784733425Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784759986Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784766266Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784774226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784781726Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784787806Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.784795006Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:33.784893698Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"949244fe-f123-42cf-8d97-ac1a8cd73056","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized)]","latency_us":535250,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:33.785151022Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.709475568s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"949244fe-f123-42cf-8d97-ac1a8cd73056\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"949244fe-f123-42cf-8d97-ac1a8cd73056\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:37.59017739Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.805878021s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.474866403Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.475172138Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.475179158Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.499104602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.499168873Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.523214879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.52329268Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.544341849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.54439771Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.565547341Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.565603612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.586873404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.586933515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.609187533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.609249944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.63007371Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.630133801Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.652432359Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.65249497Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.6735619Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.67362891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.694525007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.694581578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.716846876Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.716913967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.738053618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.738121729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.759523433Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.759587065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.779683059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.779738109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.799488768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.799540479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.817957637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.818025778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.838157553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.838212884Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.857649448Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.857714339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.878512804Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.878564155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.898257873Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.898321924Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.898522467Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.898660569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995160908Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995229869Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995238739Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995248479Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995256399Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995264499Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995277199Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99528553Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99529226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99529902Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99530664Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99531314Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99531957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99532596Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.99533352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995361151Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995370791Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995378841Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995385151Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:41.995482593Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5b7aa315-0b2f-41a2-8c57-04f43b05d439","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized)]","latency_us":521891,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:41.995733537Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.453951645s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5b7aa315-0b2f-41a2-8c57-04f43b05d439\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5b7aa315-0b2f-41a2-8c57-04f43b05d439\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.430631364Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.431074691Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.431084381Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.453566413Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.453613633Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.473477774Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.473522165Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.493457536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.493512907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.512559015Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.512603526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.531568292Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.531607713Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.549860468Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.549894579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.568845915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.568880836Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.587191652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.587228222Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.605849083Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.605891844Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.623767084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.623799594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.642055669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.6421083Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.660306035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.660348026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.678183334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.678221945Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.696384879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.69642118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.714653605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.714709805Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.732549084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.732586605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.751570052Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.751621543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.769877748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.769925469Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.788468149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.788512269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.806851846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.806895867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.80709981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.807223182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902596223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902665544Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902671424Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902697654Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902707169Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.90271556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902725272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902732322Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902740591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902749596Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902759336Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902768696Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902776226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902783596Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902789406Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902796786Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902811206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902818926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.902841147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:49.902866577Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"12ac97da-74d9-4037-b2bd-5d298ee4f320","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized)]","latency_us":473331,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:49.903105711Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.924271937s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12ac97da-74d9-4037-b2bd-5d298ee4f320\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:43:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12ac97da-74d9-4037-b2bd-5d298ee4f320\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:50.851463438Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:43:50.851526849Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"df71f341-f0f4-42b9-be1a-b5a9704f36e2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53684","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:43:55.384462657Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.38697237s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.796504453Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.797293546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.797301476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.819723556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.819798178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.839372144Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.839451765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.859194444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.859260465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.877800695Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.877857025Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.896480817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.896552558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.914001441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.914047311Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.932140474Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.932202205Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.949868741Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.949935622Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.967947744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.967998395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.985856734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:01.985921135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.004292141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.004353582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.021763515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.021834526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.041713667Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.041768067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.059747699Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.059804569Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.077836941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.077889832Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.097660201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.097719182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.117324719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.11738994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.1372403Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.137293771Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.158046305Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.158106806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.176820339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.17690085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.177166944Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.177305766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271594221Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271662922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271680022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271689952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271695602Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271702042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271709453Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271724333Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271730843Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271735963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271742223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271748953Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271754333Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271760723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271767143Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271772594Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271776594Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271795514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.271801044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:02.271861285Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"695a638e-1a03-40a9-8210-0a587787a5a6","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized)]","latency_us":476494,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:02.272059198Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.229721162s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"695a638e-1a03-40a9-8210-0a587787a5a6\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:44:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"695a638e-1a03-40a9-8210-0a587787a5a6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.483169267Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.484033011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.484045421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.506956249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.50700822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.525898485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.525961906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.545268528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.545329399Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.563768707Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.563819118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.581585436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.581642147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.599784701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.599853772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.618088047Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.618136028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.635665551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.635725612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.654594818Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.654661199Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.673566634Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.673619745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.692216366Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.692263286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.70975165Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.709802641Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.728024066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.728083067Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.74619778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.746266721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.765008924Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.765060045Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.784478288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.784536429Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.803209271Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.803260682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.821390255Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.821444506Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.840640656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.840689947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.859976859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.8600568Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.860271563Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.860418725Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955586352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955666145Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955676265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955682985Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955689555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955708805Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955717455Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955730886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955739316Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955745946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955753956Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955760596Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955766996Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955772256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955777407Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955782407Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955794467Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955800977Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.955808937Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:09.955872888Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"da13413f-44a4-4507-a961-08691d2ef5fa","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized)]","latency_us":473856,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:09.956132372Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"13.329914481s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da13413f-44a4-4507-a961-08691d2ef5fa\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:44:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da13413f-44a4-4507-a961-08691d2ef5fa\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:15.751550148Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.730413384s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:20.804201502Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:20.804284925Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7679be0e-9493-47b8-b543-86010b48894d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44557","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.251141607Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.251749237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.251764967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.275612277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.275663621Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.297254868Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.297311819Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.317439994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.319940743Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.337364136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.337422966Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.355517879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.35557393Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.373936767Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.373978948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.393010735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.393058246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.411276741Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.411365342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.429502566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.429554957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.447789962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.447848093Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.466416243Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.466481464Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.48414396Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.484203901Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.501801596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.501855977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.520346586Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.520413967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.538124744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.538185195Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.555874042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.555929763Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.575028061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.575094762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.593854726Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.593908547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.612508877Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.612569878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.631529675Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.631596096Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.63185081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.631987662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729697899Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729754271Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729762811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729777381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729792691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729799321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729805821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729810441Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729816761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729822032Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729827352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729832452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729838932Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729844072Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729859472Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729869122Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729875882Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729880973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.729893283Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:23.730015624Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"55546054-f1eb-4170-bd8e-1900dd6ccb76","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\", TimeNotSynchronized), (\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\", TimeNotSynchronized), (\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\", TimeNotSynchronized), (\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\", TimeNotSynchronized), (\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\", TimeNotSynchronized), (\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\", TimeNotSynchronized), (\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\", TimeNotSynchronized), (\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\", TimeNotSynchronized), (\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\", TimeNotSynchronized), (\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\", TimeNotSynchronized), (\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\", TimeNotSynchronized), (\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\", TimeNotSynchronized), (\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\", TimeNotSynchronized), (\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\", TimeNotSynchronized), (\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\", TimeNotSynchronized), (\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\", TimeNotSynchronized), (\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\", TimeNotSynchronized), (\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\", TimeNotSynchronized)]","latency_us":479949,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:23.730244108Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.606211676s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"55546054-f1eb-4170-bd8e-1900dd6ccb76\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 18:44:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"55546054-f1eb-4170-bd8e-1900dd6ccb76\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 8bffd25b-2901-4312-b7a6-dab1af509a50, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ef6fb729-d03b-4f78-95a1-df1b8fbfe594, kind: Service { id: 8bffd25b-2901-4312-b7a6-dab1af509a50 }, name: Name(\"ntp-8bffd25b-2901-4312-b7a6-dab1af509a50\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 120])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: c1bda75f-7bfe-4403-94af-34867f337f31, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 70f181a1-2827-4ca2-aa37-744b188fafb8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 808f488e-12f3-4bfc-aef1-cad905bec1af, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a18f061-72f1-453b-bd45-d62bee284167, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 74c049e4-0204-41c5-9eae-1ea9332427b8, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b2d12000-0534-43f3-aecb-025d60088a7f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f1a0cefe-529f-44f2-8ccb-2a5998075f57, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External } } } }, OmicronZoneConfig { id: 132eb58b-4be6-480a-b421-ec59de440f35, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2740376d-d5de-463d-a8d1-2645473ef0ea, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External } } } }, OmicronZoneConfig { id: a4d98cba-76cd-46df-bcc9-c060da715f68, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External } } } }, OmicronZoneConfig { id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c508b20-ea60-43f1-bd87-4593851d3b40, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External } } } }, OmicronZoneConfig { id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.306442632Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.307309816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.307324606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bfbfbed2-0c7b-46ae-9a85-844e2c2af98e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.327292228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.327327769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d888dc5c-8f48-4f16-ba5f-c2760b3610b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.346504288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.346542949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5ac60829-580a-40c1-8bb7-8a2b487569a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.364461479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.36450025Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1bda75f-7bfe-4403-94af-34867f337f31, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.384053295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.384089176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 70f181a1-2827-4ca2-aa37-744b188fafb8, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.402213219Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.40225176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 61cb6caf-7127-44ff-b97d-23c7d6df94f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.420917952Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.420960322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 808f488e-12f3-4bfc-aef1-cad905bec1af, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.43808975Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.438126281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8091f6e2-58ac-4ea2-be97-ffe07a6d4402, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.457073117Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.457112168Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.476013863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.476071084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8b280afb-d117-4f80-a3f2-2835451a0bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.493630549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.493682059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8badc56b-0df0-462d-9438-dbb64cfc40f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.510860178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.510912309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 5da902d8-66b7-4991-9f0f-d9408ae4cf51, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e607437-7489-43a3-8665-a0998816c2da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.529168684Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.529212265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 132eb58b-4be6-480a-b421-ec59de440f35, dataset_name: DatasetName { pool_name: ZpoolName { id: 9792c5a6-e0f4-4250-91d8-71386ac84474 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.547045824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.547083964Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2740376d-d5de-463d-a8d1-2645473ef0ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 61f4fa9a-1baa-4a1b-8076-77046f3a1be7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.564979144Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.565017665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a4d98cba-76cd-46df-bcc9-c060da715f68, dataset_name: DatasetName { pool_name: ZpoolName { id: 3481a5aa-d464-43f4-8c3b-e7c18df1676b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.583455423Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.583494594Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 223e9a1e-d516-47f3-a9b7-67967eba10d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.601101199Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.60113663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7c60b212-55fd-4f18-a7f0-5fc2accc5f4d, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb75e0c-fc60-474c-85f7-e6b93e4d0425 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.619551017Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.619602618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3c508b20-ea60-43f1-bd87-4593851d3b40, dataset_name: DatasetName { pool_name: ZpoolName { id: 2db534d5-2c49-4cc7-bcdd-c78bfddad2ae (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.637793963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.637835703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 56328015-6e5b-46f4-8b73-bfdc206d32b1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.655277596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.655310517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ad836a09-f39e-4c1c-b240-e7a9cb6917ef, dataset_name: DatasetName { pool_name: ZpoolName { id: 59a6abe1-72b0-47e0-8dd6-231a2ecca4d7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.673642473Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.673688704Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.673953998Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.67409883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.770779152Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.792779166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.7930274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.793154472Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.79429099Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.815710204Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.815922988Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.815977998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.84040187Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.840446301Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:35.840451261Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.006300824Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.006520347Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.006545547Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.006563348Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.006586298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.034729798Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.034783619Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.034788369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.035374478Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.041024916Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.047332185Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.052583247Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.05787036Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.062424551Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.066946712Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.208576386Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.213559364Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.219308694Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.223420008Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.22736045Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.2318708Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.236540328Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.241872367Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.24656809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.251745001Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.257475717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.263398563Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.268697516Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.274468065Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.281887512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.288735359Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.29454821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.30029183Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.307194358Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.313663429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.319413189Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.381664802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.461770895Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:36.463548233Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"18.251351451s"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.636415275Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.691056699Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.754279768Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.756109236Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.756123639Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.810411626Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 134, 201]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.810458116Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.812152173Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.812169903Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.864375619Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 248, 167]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.86440883Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.866200538Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.866216738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.918850451Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 244, 150]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.918881271Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.920162771Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.920176981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.968354294Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 255, 147]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.968394885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.971343331Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:36.971360321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022096645Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 244, 60]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022127265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022345739Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022358509Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022420275Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022432144Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.02244702Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.02245348Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022498711Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.022506821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.027302466Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.028532065Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.083506795Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.086671184Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.086901058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.086916748Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.087135502Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.09083276Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.094818682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.094832242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.142286214Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.148574782Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.249026043Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.26032465Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:37.959146745Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.065578339Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.081187083Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.147129244Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.156675123Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.170809134Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.178813179Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:38.195187825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.847618177Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.856661188Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.856842331Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.883887933Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.889703074Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:43.893836818Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.060999652Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.061341518Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.102076124Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.103226702Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.119482247Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.120782937Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.139483019Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.140875381Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.160103174Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.160352686Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.173741815Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.177434243Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.205245237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.207432442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.208883904Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.209078497Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.221826637Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.234949252Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.417995274Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:44.418674174Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 71.411466ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.515644821Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 50.490155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.531190704Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 53.009331ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.544562613Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 72.092753ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.561010092Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 49.716985ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.575188352Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 64.974156ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.588670792Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 49.724841ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.602865514Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 80.19787ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.637887022Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 28.111716ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.664915564Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 68.373058ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.692163661Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 88.889708ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:44.799625421Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 74.82048ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.06442345Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 91.337424ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.087631801Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 100.801316ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.11242224Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 294.737243ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.141372303Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 149.945879ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.194428242Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 44.800955ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.234412338Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 75.700862ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.266248045Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 50.005003ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.291896566Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 118.931216ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.323143295Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 51.199463ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.610465017Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 202.311799ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.619543139Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 149.100432ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.648258578Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 65.123912ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.780230601Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 36.414346ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.858861081Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 251.947214ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:45.95610403Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 120.784313ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.001640522Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 358.513507ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.034290243Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 151.408926ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.060835688Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 56.176414ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.091366985Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 105.53706ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.115912259Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 53.613232ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.132307025Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 188.163554ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.149310461Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 276.137963ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.164555989Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 279.160774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.178942234Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 168.528219ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.191956698Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 265.626955ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.204162189Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 134.950108ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.216699785Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 273.800851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.231914313Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 541.590794ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.244190714Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 280.483272ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.255668314Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 181.31884ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.267492959Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 383.566226ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.277589117Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 244.171507ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.288079271Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 305.157704ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.354156954Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 208.969999ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.366311944Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 484.540231ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.377727062Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 941.582727ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.405332744Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 236.938771ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.454504053Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 285.26672ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.471236444Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 688.841667ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.483822892Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 455.993896ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.500302339Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:46.500865068Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.097938453s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.522678189Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:46.522809591Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 425.629709ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.545351943Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.19985287s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.557791488Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 481.456848ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.593031949Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 939.2601ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.67816216Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 805.66791ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.69611097Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.109132346s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.710499975Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.165143551s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.771171944Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 70.267399ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.785295725Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.05826399s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.799341604Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 50.89715ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.868490475Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.022344724s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.881651971Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 192.64147ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.933080715Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 589.373326ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.968657231Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.077652021s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:46.983901189Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.19832805s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.087725222Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 344.0099ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.14451383Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 667.616653ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.18736037Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 675.629519ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.361196078Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.199128525s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.509361795Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 893.984752ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.549016275Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.209557213s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.587308563Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 869.853917ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.638786028Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 910.935059ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.656966573Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:47.842797178Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 821.614766ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.915010827Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 573.879212ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.929397721Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:47.929648485Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 664.270951ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.950357929Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 546.773725ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.963327252Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 802.591505ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.975689145Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 968.074059ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:47.989304828Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:48.000015256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:48.000159018Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.31762832s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.048964461Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 879.264372ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.073431643Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 46.603409ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.193756385Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 33.964173ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.227878778Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 138.905134ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.25102769Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 77.220637ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.277530874Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 689.399057ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.296371559Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 122.525836ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.366656698Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 125.309092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.404151354Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 504.824502ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.452385528Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 206.814484ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.518285248Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.14054438s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.535091001Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 513.68212ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.567948475Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 956.831511ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.595363954Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 518.830807ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.607337031Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 762.511806ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.618325913Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.010993531s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.630213329Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 629.307687ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.718923105Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.132855768s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.782864575Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.065360877s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.794933234Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 880.415612ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.80683517Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.385996882s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.818565123Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 891.245283ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.96359845Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 563.233724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.975868592Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 660.521167ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.986895345Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 514.545156ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:48.99937348Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 956.999927ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.094896383Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.109285605s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.138550026Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 898.926356ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.359733964Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.182029844s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.37227211Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.10138097s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.389964737Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 811.820264ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.526550432Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 635.014231ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.549965558Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 780.517651ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.562786738Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 986.920706ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.653213782Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.477786743s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.665534095Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 717.469743ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.685626809Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.266768554s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.698530301Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.496775587s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.86868838Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 651.791287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.881824876Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.019066936s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:49.926636877Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 873.605659ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.063855372Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 636.692542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.196294053Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.416331056s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.214818372Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 522.565835ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.259106215Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.352611818s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.272957141Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 641.594497ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.349109342Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.359377203s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.360688873Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.211543747s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.413667481Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 832.907339ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.501579726Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.147636975s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.543975819Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 571.687189ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.564100893Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 720.918495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.650909781Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 610.080849ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.793390088Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 975.338828ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:50.847466554Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.143862489s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.239197677Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 530.635283ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.25282557Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.25650003s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.265227774Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.29517742s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.277518535Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.050796424s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.291000125Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.367265974s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.30320691Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 519.659822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.347170881Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.161619277s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.37727677Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.306385319s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.389227437Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.264009995s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.415306963Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:51.458841502Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:51.501537868Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f90dae6c-e578-4efe-8e25-36219ab57207","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55229","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":41689,"response_code":"200"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 647.014226ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.637789643Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.12356782s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.650024774Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 741.52677ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.661927859Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.291959472s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.705400277Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.42785369s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.731625146Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 626.106431ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.794031989Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 807.757631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.832252895Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.2394119s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:51.878737599Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:52.284420234Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:52.284653858Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.477490559s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.298928661Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 674.558161ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.515438057Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 950.972545ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.524780082Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.212782415s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.532788047Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.053632798s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.540581709Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 60.82403ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.549284484Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.067452848s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.557602374Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.129520304s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.566482793Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 573.21542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.707597373Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.112659264s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.716501562Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 628.212547ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.724197572Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 601.39775ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.731407684Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 149.450291ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.741292018Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.075301195s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.749176481Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.381632509s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.78178723Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 135.778058ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:52.900113994Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 628.836849ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.006717216Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 574.339674ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.044602817Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.388185051s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.127222165Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 805.412672ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.168455178Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 689.888069ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.198970264Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.368461185s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.290955658Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 814.216236ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.341959093Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.076585547s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.362433213Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 864.072816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.484817461Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 647.043724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.603802166Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 823.102676ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.62775282Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 671.670684ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.63611272Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 669.340112ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.644653633Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.113958667s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.704843082Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.124384085s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.756013799Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.18711104s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.78557996Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 896.22015ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.833559169Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 710.552998ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.841918729Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 611.735748ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:53.898030483Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 709.096594ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.024020828Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.037532869s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.166205915Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.414401725s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.174829649Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.178964798s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.260316116Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 554.149509ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.316890624Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.247322871s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.325381027Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.097755695s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.357481977Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 556.757967ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.447180456Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 550.239183ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.459727842Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 972.105471ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.518738212Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 653.931022ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.52697183Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 853.579932ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.562062287Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 627.9757ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:54.668286123Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 970.131869ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.2625247Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.22727264s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.270474174Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 568.080406ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.278467149Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.058618449s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.286492804Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 972.933878ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.295156329Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.494178198s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.303963196Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.220530865s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.312208985Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 759.002666ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.321393018Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 838.468267ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.338952372Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 805.959517ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.360369616Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 552.569151ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.378167274Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 676.846492ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.391609583Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.024866182s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.401827392Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.272763374s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.41384704Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.070152306s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.461592634Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 787.424333ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.470051536Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 855.442559ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.478040521Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:56.479388906Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:56.479444847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:56.479483285Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:44:56.484304438Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 37.586757ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.650814175Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 46.215915ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.658015127Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 142.459947ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.698567639Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 84.006197ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.712652149Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 245.351267ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.806464882Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 194.855028ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.851179049Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 534.977189ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.860428963Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 993.887273ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:56.939537996Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 350.794243ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.054887114Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 582.729479ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.066925162Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 648.093478ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.078860998Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.407487777s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.089402403Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.438846938s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.176018003Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 525.84106ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.186388155Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 805.600812ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.242069993Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.125236984s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.267208795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 580.139538ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.276772544Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 769.9458ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.343759419Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.236804813s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.354279393Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 736.742681ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.404882952Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 730.537665ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.415437726Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.096995261s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.434989741Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.343051589s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.508232263Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 631.450353ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.54198652Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.359461951s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.551140352Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 526.965077ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.660844503Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 566.146743ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.696878495Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.276444078s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.72094233Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.463176229s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.735780581Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.416510016s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.806646226Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 959.797371ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.865124355Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 936.471251ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:57.942729908Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.300611681s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.056589263Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.08786291s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.122520391Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 529.968125ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.15067812Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 834.519666ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.158142056Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.375880756s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.183476162Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 522.02976ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.196097978Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.216044257s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.271116208Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 999.863742ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.401957348Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 715.498539ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.505895769Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.250897877s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.542106503Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 632.408568ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.601239205Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.223208967s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.62398632Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 668.031517ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.690873943Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 725.184082ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.72786902Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.299056296s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.834193178Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 918.68353ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.859081426Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.404539737s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.887821423Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.21266501s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:58.918472201Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.473820987s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.002663634Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 815.442885ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.010822101Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.161651625s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.22176439Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.340160796s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.238067414Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.378889767s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.248876683Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 633.189006ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.256583483Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.305571921s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.265423251Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.416408127s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.366840493Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.434349438s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.375555378Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 549.441015ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.411720022Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 881.114987ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.46160017Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 956.664534ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.496679617Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 805.123155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.568858092Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 921.858289ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.788091281Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.156648153s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.801879236Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 573.878219ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.835118094Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 851.812851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.856772112Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.285090985s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.89969742Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 530.776162ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:44:59.96955282Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.215586205s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.140859901Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 941.263009ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.148278526Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.321707273s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.30181834Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.268114269s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.353361634Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.462246475s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.382955316Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.308674763s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.391615081Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 929.748503ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.418092164Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 705.113155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.46275992Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.05892424s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.485667117Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.35372054s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.50956822Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 825.672016ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.580564067Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 647.745822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.589236732Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 561.166975ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.638793755Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.04829871s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.719632365Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.231918883s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.728215699Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 960.549807ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.793532018Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 842.798577ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.819602984Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 987.095415ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:00.970257662Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 849.847245ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.100794438Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 746.073214ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.185800143Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.049215597s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.197619038Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 747.994163ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.222627288Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.390895617s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.248738415Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 685.90882ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.36197237Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 520.61535ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.372678527Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 571.520504ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.416724444Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.002362985s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.557635811Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 829.754497ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.633026027Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.347977044s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.643705993Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 541.724177ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.673475697Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.315221949s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.7108505Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.282291113s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.765255998Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 577.182342ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.777807934Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.112945107s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.855645298Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 936.173809ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.872883826Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 641.387453ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.903110457Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.251124794s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.94238661Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 686.965208ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.95972229Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.224568583s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.968775881Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.408691501s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.977469457Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 701.13563ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.986930974Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.26959245s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:01.998776659Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 529.464176ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.059347973Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.431561835s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.226108503Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 850.626228ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.25796722Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 557.428615ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.365441656Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 829.015455ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.47533834Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 701.588114ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.555841695Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 867.138191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.570872419Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.178528629s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.598635322Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.157048348s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.64984317Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 777.334348ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.660134231Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 750.418673ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.699617947Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 519.464786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.819424445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 712.766431ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.933183538Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 626.077964ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:02.979084603Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 658.36774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.002912765Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.22172149s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.036113623Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 913.383854ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.059483927Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.337903981s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.118560258Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 788.843516ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.203047336Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 589.87725ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.212198118Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.229881665s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.269259418Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 974.257034ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.278089306Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.272372712s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.31621502Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 900.765786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.349786744Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 662.480059ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.396754946Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 538.658275ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.447629039Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.332578765s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.456058961Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 946.011201ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.465250884Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.393721727s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.615101951Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 892.919162ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.656931113Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 685.282113ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.667086261Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 721.998951ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.67597998Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.286259383s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.787152173Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.195729292s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.812155233Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 775.856349ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.820633876Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.276930525s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.983823159Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.275474838s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:03.995282828Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 836.865392ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.003553987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 844.625286ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.068216595Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.292212447s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.262007847Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.441968441s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.271371453Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.054860461s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.283467552Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.172590007s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.362188139Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 656.946905ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.408487961Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.256253179s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.419423401Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.078147461s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.466190441Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.060647151s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.508203226Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 961.175382ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.559531226Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 946.910347ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.598011216Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.350786985s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.605679525Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.007464881s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.799330235Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 619.955307ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.850872469Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.133485699s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:04.923580712Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 705.569076ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.018075495Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.04851018s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.026833602Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 698.715126ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.075334941Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.373863297s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.083498905Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 706.502499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.270785546Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 697.144021ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.280220013Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 926.074091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.347973879Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.283132078s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.480722589Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 692.19219ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.53014958Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 664.757265ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.545311666Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 926.104941ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.553768758Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.017720326s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.561422067Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 841.414707ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.568791502Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 988.728386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.578110578Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.156048409s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.686161072Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 911.262367ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.722864395Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 736.125392ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.732597566Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 718.078184ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.783879286Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 786.347514ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.817239206Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 702.696735ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.969204665Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.431613745s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.98751781Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 787.55217ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:05.997523306Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 730.657933ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.06954192Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 730.92314ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.087084263Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 518.565785ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.221311576Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 896.57362ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.232438849Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 692.879822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.287340305Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.147503258s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.421429886Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.104118538s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.468920187Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.340485239s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.480257473Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.21201786s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.492793309Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 956.757889ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.512839732Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.429756555s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.57816254Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.480916448s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.5916305Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 646.043269ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.617462783Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.118828749s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.645353088Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.264488443s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.684671071Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.132821855s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.753022147Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 946.766712ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.776623535Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 708.709198ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.795957406Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 899.783659ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.811130493Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.445736074s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.828660206Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.066144645s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.852354286Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.471474374s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:06.990600341Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 718.727882ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.142883795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 543.172251ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.277204269Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 764.246018ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.43180729Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.029991509s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.480387098Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.220663357s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.517246642Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 861.003914ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.579124447Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 618.554619ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.587397626Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 906.285609ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.717688727Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.461580931s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.729237228Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.407030366s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.744929572Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.27860129s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.774812568Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.46654472s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.83007304Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 531.320949ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.838201987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 977.450365ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.872260167Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 522.651648ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.894000066Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.185135994s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.926750417Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.485058529s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:07.958968889Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.067383534s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.022066973Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.150167053s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.087472913Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.401421217s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.207104778Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.393233309s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.222332676Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 600.235931ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.285773815Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.476911816s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.380301469Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 928.083894ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.433947465Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 526.900085ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.456525577Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 850.324136ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.477684957Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.211708324s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.524717831Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 827.554851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.633915243Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.271332263s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.751113921Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.319412038s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.864306561Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 1.242588455s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.90053734Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.236991144s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:08.995595142Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.065793822s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.065230818Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 514.044997ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.101783038Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.098770649s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.123843362Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.206046952s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.161805754Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.416546475s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.200775851Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 973.480306ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.247175795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.271885237s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.306548891Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 691.731135ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.337752437Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.499305889s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.373185999Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.04377276s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.455081487Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.094386809s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.472179873Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.075646843s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.620354873Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 685.120155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.630017954Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 1.470061605s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.639540453Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 978.016547ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.748253668Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 653.409581ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:09.872367832Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.337763345s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.035235832Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 519.502251ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.046518858Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.18585692s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.142036587Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 878.263826ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.155647069Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.234909186s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.199185498Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 653.183197ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.233627295Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 535.892851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.246582357Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.087344991s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.261549891Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 641.168879ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.334970635Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 1.200528215s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.404149814Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.477647921s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.544015075Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 908.054736ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.582861211Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.479960461s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.605814899Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 800.807958ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.623058338Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 648.01309ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.640647972Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 560.327647ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.661409965Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 598.02574ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.719688614Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.271766167s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.748114017Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.306428947s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.799205034Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 1.418218468s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.893515264Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 542.603967ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.915451616Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 529.813631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:10.991619124Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 900.3883ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.052812508Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68\") failed: Property not found. retry in 866.921634ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.128705491Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 786.777414ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.238528344Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.255942545Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"28.067414905s"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 738.647876ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.307079593Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 544.919799ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.339833433Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167\") failed: Property not found. retry in 1.300546982s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.358985992Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.278613537s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.372776847Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.227955033s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.389810293Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 588.004779ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.437640388Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d\") failed: Property not found. retry in 1.0060208s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.450081532Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 597.83622ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.470296448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.476789718s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.506687455Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.040785795s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.533218659Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 896.289658ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.62113969Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.460089259s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.900034297Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4\") failed: Property not found. retry in 716.150928ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:11.967410558Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 572.604509ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.032230449Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 870.931108ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.045580987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.413125244s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.0579475Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 602.952878ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.071728435Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 763.803972ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.088271823Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.354286295s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.102454734Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.122643584s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.125129887Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 908.673582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.139606293Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0\") failed: Property not found. retry in 850.693603ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.327989281Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 874.681001ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.534539381Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.025593193s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.590757238Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.329576947s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.624207639Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8\") failed: Property not found. retry in 1.193413335s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.640147508Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f\") failed: Property not found. retry in 1.07226843s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.66466451Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 846.278171ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.730624489Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.134096257s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.876092466Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea\") failed: Property not found. retry in 1.381313765s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.933950228Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.168998347s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:12.999425899Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 1.032299036s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.063068431Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\") failed: Property not found. retry in 1.495356323s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.259340552Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 1.169994711s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.370926792Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57\") failed: Property not found. retry in 651.190529ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.42275647Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2\") failed: Property not found. retry in 1.197047017s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.469924755Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef\") failed: Property not found. retry in 1.196865363s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.484444492Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f\") failed: Property not found. retry in 1.1438487s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.589236486Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a\") failed: Property not found. retry in 1.091554757s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.629399012Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40\") failed: Property not found. retry in 1.347766383s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:13.964544007Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35\") failed: Property not found. retry in 1.128944177s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:14.022710314Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f\") failed: Property not found. retry in 593.957165ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:14.107419175Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51\") failed: Property not found. retry in 1.058991982s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:14.179306356Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9\") failed: Property not found. retry in 904.912542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:14.554828481Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.50594458Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_external_dns_8b280afb-d117-4f80-a3f2-2835451a0bc9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506065602Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_2740376d-d5de-463d-a8d1-2645473ef0ea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506089223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_223e9a1e-d516-47f3-a9b7-67967eba10d0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506095893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_1d5b81a3-381f-4d1b-b975-02e6284ad71f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506101853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_5da8cc16-12a1-4dfc-8dc7-6845077fc9c5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506110723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_74c049e4-0204-41c5-9eae-1ea9332427b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506116853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_132eb58b-4be6-480a-b421-ec59de440f35"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506124263Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_external_dns_94a2c9b0-295c-488e-99e0-bfb110e44e3a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506130054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_oximeter_3a18f061-72f1-453b-bd45-d62bee284167"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506173424Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_a4d98cba-76cd-46df-bcc9-c060da715f68"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506183774Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506196335Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_3c508b20-ea60-43f1-bd87-4593851d3b40"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506211305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_ad836a09-f39e-4c1c-b240-e7a9cb6917ef"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506222645Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_f1a0cefe-529f-44f2-8ccb-2a5998075f57"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506242705Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_5da902d8-66b7-4991-9f0f-d9408ae4cf51"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506254035Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_3662ecea-b03d-4e92-9b8c-8abcae1826e2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506264885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_clickhouse_8badc56b-0df0-462d-9438-dbb64cfc40f4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506677952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_b2d12000-0534-43f3-aecb-025d60088a7f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.506707532Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_c1d00ceb-53a6-4a90-afc1-dfd624c4bb5f"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.507057228Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.510996579Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.51680798Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5d050d39-b314-4fee-a1fb-7a49101e64c8","remote_addr":"[fd00:1122:3344:101::1]:50948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":40252389,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.5328229Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.533177365Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.533409259Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.541555726Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.550007188Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:15.590893925Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:16.318706883Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:16.379495361Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:16.556882677Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:16.749582222Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:16.954937603Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:17.114327338Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:17.447821878Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:17.640150317Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:17.828581915Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:18.422045778Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:19.124473646Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:20.053252151Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:20.809796557Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:20.951624228Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:21.111228767Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:21.541256322Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:21.541872942Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a4d16f86-6ecb-49c8-aeaf-e63e3b75c07c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49316","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:22.089360417Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:22.12098878Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:22.261827626Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:23.124709529Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:23.381301109Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:24.128004191Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:45:24.261204428Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:24.851108556Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:25.131342934Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/447ca897-407e-4dad-a0e2-4f6036da9497): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T18:45:26.144015833Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51fcf300-ade0-4275-ae29-875fa51400c4\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 18:45:26 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"054a3cb3-27d1-4db0-b7f6-bb5f5aad2e14\\\"\", request_id: \"51fcf300-ade0-4275-ae29-875fa51400c4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:26.92128183Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.283939676Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.37034984Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36308"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.372510837Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.455167115Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"f7efcc37-f5f9-4cda-aec4-561897a22d67","remote_addr":"[fd00:1122:3344:101::a]:36308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84654,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.456173391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.456225562Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.457826697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:28.458106381Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"88bf4ac9-e187-4129-b110-97db5b00221b","remote_addr":"[fd00:1122:3344:101::a]:36308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2676,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.654301401Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49265"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.659262808Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660218843Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660257364Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660264894Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660288644Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660305624Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660309585Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.660316065Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7f0a0b64-a137-468d-97eb-481da00076b7","remote_addr":"[fd00:1122:3344:101::a]:49265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":5812,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.662476904Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.889791108Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46825"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.889869756Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.97817656Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"0b2cd16d-e3fe-4374-807b-280bcdbe2161","remote_addr":"[fd00:1122:3344:101::a]:46825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88285,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.978452564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.978500875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.97881434Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:29.979079284Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"958563d7-77cd-40c9-a5db-adfc5147ccab","remote_addr":"[fd00:1122:3344:101::a]:46825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":702,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.404976154Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52728"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405729516Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405771607Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405813198Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405830098Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405839678Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405862938Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405868168Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.405872938Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"974ebee1-6fd4-4240-8845-d37c0420f024","remote_addr":"[fd00:1122:3344:101::c]:52728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":846,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445365194Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36057"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445588087Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445638608Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445673229Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445687719Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.445703589Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.44572486Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.44573913Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.44575481Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"aed970d7-d042-47f5-a9d9-46ca77e3eeec","remote_addr":"[fd00:1122:3344:101::a]:36057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":281,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543069107Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59479"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543301681Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543357032Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543386283Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543399193Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543420483Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543435803Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543453093Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:30.543471274Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6614c8eb-d94f-42df-a339-da0136467776","remote_addr":"[fd00:1122:3344:101::b]:59479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":310,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:34.822978845Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:34.8239409Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bc4b641f-1224-491e-8a7f-9a4a27f1767f","remote_addr":"[fd00:1122:3344:101::a]:65474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":929,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:34.894822187Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"375b1b0c-4f01-4178-9f04-59b684b25131","remote_addr":"[fd00:1122:3344:101::a]:65474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3000,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:51.580778623Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:51.581288321Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c9f7705b-803e-4a91-bec4-cee7514f7ac2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54769","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.055672011Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.055820603Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0f9ac98d-b830-4201-810b-0058f567564f","remote_addr":"[fd00:1122:3344:101::c]:51914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.065696147Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48087"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.065770188Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"137aecee-5d5a-4262-85c8-830c1bf12074","remote_addr":"[fd00:1122:3344:101::b]:48087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.077416493Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4e763b5a-0c19-4072-9ef5-91ca5f25bc8e","remote_addr":"[fd00:1122:3344:101::c]:51914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.088055286Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4b9d16b6-05d8-4875-9865-f428ca286106","remote_addr":"[fd00:1122:3344:101::b]:48087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.132758163Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.134599951Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e5cf6999-2888-4344-b712-1c843b8b7582","remote_addr":"[fd00:1122:3344:101::b]:57844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1799,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.262912642Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49541"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.263044064Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9900785f-e9cb-4701-b853-026745593abc","remote_addr":"[fd00:1122:3344:101::c]:49541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.288283848Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62963"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.368221654Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6c178908-8a33-449c-9346-c527cedfbab6","remote_addr":"[fd00:1122:3344:101::b]:62963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79853,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.368347946Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.420499939Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"00096115-9c61-454f-8464-bb5dedd4466b","remote_addr":"[fd00:1122:3344:101::c]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52076,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.421739519Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1a3a4d00-4249-481a-a94c-df50a881bbda","remote_addr":"[fd00:1122:3344:101::b]:62963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1088,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.421959342Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9220079d-9903-4668-8cd8-8320a1820739","remote_addr":"[fd00:1122:3344:101::c]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":136,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.445977206Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.446010407Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.446904121Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.447622652Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.448251452Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"db33556e-4ad4-440d-8809-430e8cbdea24","remote_addr":"[fd00:1122:3344:101::b]:62963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3505,"response_code":"204"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.449013114Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/92e7db57-0bba-4f49-937c-e777282668a4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.449038194Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.449187176Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.449284788Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.449314978Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"80bb6769-6002-41a6-bef9-e39755bd1342","remote_addr":"[fd00:1122:3344:101::c]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":414,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.810073963Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64967"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.810851435Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1c89a550-d19e-4ffa-9582-6280a5f87551","remote_addr":"[fd00:1122:3344:101::a]:64967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":754,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.818562286Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"fe9fc81b-5fdd-4fb1-bb82-07d0b5e94e3b","remote_addr":"[fd00:1122:3344:101::a]:64967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.906478926Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:57.907204637Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9a491216-ff09-43d8-8f80-7eac4fa5cef2","remote_addr":"[fd00:1122:3344:101::a]:60201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":707,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:58.039590541Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:58.08889084Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d4b81c02-d454-4d44-a7f0-8f04f95dd397","remote_addr":"[fd00:1122:3344:101::a]:57346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49236,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:45:58.089833404Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"95b56fc9-1964-42ee-99ca-a68cb60cef16","remote_addr":"[fd00:1122:3344:101::a]:57346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":731,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:09.432280613Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:47370"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:09.443561129Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"d185fd57-3aee-45bc-94bb-09593306ec30","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":11055,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:21.616712443Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54923"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:21.616804204Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a0296f38-5560-490e-9497-87b0766d46b7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54923","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.16635825Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.166546393Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"987beedd-19d0-4d38-a4c3-73976ac48090","remote_addr":"[fd00:1122:3344:101::c]:41041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.173917118Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.174006049Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c3113c2f-5e45-4914-affe-2472a32e275f","remote_addr":"[fd00:1122:3344:101::b]:44735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.182223367Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7ca7da44-c9e9-4bb1-96f9-4431f9e08fac","remote_addr":"[fd00:1122:3344:101::b]:44735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":991,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.284655905Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.284730476Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c460eeea-dc8f-4edc-8f17-063b99d02d98","remote_addr":"[fd00:1122:3344:101::c]:44368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.293082586Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f6b4318d-1eff-4a55-b4bf-bc0e80dfbf81","remote_addr":"[fd00:1122:3344:101::c]:44368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.430060912Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.430236635Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"739623c2-101e-4935-9666-629c1fd0fdfe","remote_addr":"[fd00:1122:3344:101::b]:50774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.475614502Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42267"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.52616745Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"cc0d77bc-b5c5-4246-bde8-081a17d07b1e","remote_addr":"[fd00:1122:3344:101::c]:42267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50489,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.526436225Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4444dbfe-23a2-47f4-a7ee-50385b6ede0b","remote_addr":"[fd00:1122:3344:101::c]:42267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.573693031Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.632025911Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a5966d0a-5e7a-4989-b269-57bf8ecf3f72","remote_addr":"[fd00:1122:3344:101::b]:45961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58213,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:27.632385146Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"95cfbc2e-7556-47df-80eb-58f07e8836e4","remote_addr":"[fd00:1122:3344:101::b]:45961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.033946867Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57462"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.034087429Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5cc5abd9-6e72-4d1b-84d6-b94b3a3f0f73","remote_addr":"[fd00:1122:3344:101::a]:57462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.042811945Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a2ca2dc1-c1c7-40d7-af8b-417d3cc942f9","remote_addr":"[fd00:1122:3344:101::a]:57462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.08541378Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.085554152Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"217eb82f-b6fa-4f55-88da-7c8c0bbb4c1b","remote_addr":"[fd00:1122:3344:101::a]:56375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.246238067Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.297362214Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2680e5f8-e2f0-42f8-a696-bf211724e5b3","remote_addr":"[fd00:1122:3344:101::a]:34139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":51068,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:28.297603218Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0cd4427f-804a-42e9-9537-45b97899db81","remote_addr":"[fd00:1122:3344:101::a]:34139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:37.609200261Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:39.433667328Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"b937e22f-4d89-4e36-b1f7-802c23d7b05a","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":755,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:51.649130717Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:51.649756747Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"86aa5856-1e99-4c57-89b8-e4a86a4344a5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63080","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":625,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.032146577Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.032292279Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8adbbd3a-65df-492a-9088-f93a07625bb2","remote_addr":"[fd00:1122:3344:101::c]:36015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.040432736Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4603448d-9ade-49e9-99d9-eb67e730b39b","remote_addr":"[fd00:1122:3344:101::c]:36015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.083414686Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.083522658Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ed8be534-c5f2-47ed-ab3e-9f513ea97dbf","remote_addr":"[fd00:1122:3344:101::b]:47138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.091748006Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cddec2c0-8616-45a8-844e-f3a3f35630fe","remote_addr":"[fd00:1122:3344:101::b]:47138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.144912815Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.145081768Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"fcaa4591-5847-4ca8-815a-42843bdd29b7","remote_addr":"[fd00:1122:3344:101::b]:40320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.156845611Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.157723135Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4e5beb46-989d-4dc3-bc64-2c1d6c42f00a","remote_addr":"[fd00:1122:3344:101::c]:41508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":848,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.20938847Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.254795828Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8b614997-305e-460b-ab50-e1e7786dd80d","remote_addr":"[fd00:1122:3344:101::c]:49593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45346,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.255045662Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"467faab9-41a3-4126-9434-da54d75babf9","remote_addr":"[fd00:1122:3344:101::c]:49593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.298936307Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33978"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.343478711Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8979128f-7a58-4b5e-9e44-a084b8823429","remote_addr":"[fd00:1122:3344:101::b]:33978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44456,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.343753305Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"667df544-e69f-4116-963c-1852649911ca","remote_addr":"[fd00:1122:3344:101::b]:33978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.724776487Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.724914329Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"106ed184-817e-46c2-8d26-a8348dd5e767","remote_addr":"[fd00:1122:3344:101::a]:53344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.732370375Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"23fe9b1a-449e-4cfb-b709-685b785510f0","remote_addr":"[fd00:1122:3344:101::a]:53344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.789573747Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.790857767Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"6d5407b3-8489-40ca-845f-979c3bc0f1f6","remote_addr":"[fd00:1122:3344:101::a]:45640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1256,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.886229733Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53966"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.934582327Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"501cf83e-d9de-47fb-926b-01020f29c1a8","remote_addr":"[fd00:1122:3344:101::a]:53966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48293,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:46:57.934825791Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b8b9a995-4662-48b8-b6ff-8852df7b9956","remote_addr":"[fd00:1122:3344:101::a]:53966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:09.434844386Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"655de076-c1a5-4f52-9769-4ee8ab676c4f","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:21.683303809Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:21.68399038Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"736821dd-39f6-48cf-8db9-204be8aadb2d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43648","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.1773494Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.17862292Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0ea80894-e022-4024-b8b6-9b477896978f","remote_addr":"[fd00:1122:3344:101::b]:46343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1243,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.260999135Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.261088916Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"283993d0-e752-4130-8b1e-f0c47c6afcaf","remote_addr":"[fd00:1122:3344:101::b]:51807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.269925944Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a46df462-e4a7-4fa0-bd34-3b7c3680d98e","remote_addr":"[fd00:1122:3344:101::b]:51807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.316694053Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.317705399Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fb534e05-703f-44f2-9c7d-103b94ebfa6f","remote_addr":"[fd00:1122:3344:101::c]:38615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":951,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.32545564Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ad943d06-b25f-4826-99c2-ae502348f6f9","remote_addr":"[fd00:1122:3344:101::c]:38615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.3517706Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62505"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.351889502Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"edc5fb18-4377-4584-a0ec-4de0b1be8855","remote_addr":"[fd00:1122:3344:101::c]:62505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.495892437Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53250"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.543310767Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"69661717-67ef-44bc-b732-9bc5534912e8","remote_addr":"[fd00:1122:3344:101::b]:53250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47369,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.54356146Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bcd94035-3bfb-4004-913b-630856c4f921","remote_addr":"[fd00:1122:3344:101::b]:53250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.583556694Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.637454824Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"db3acfd2-e97b-49b6-ac87-5f2b49aec1dc","remote_addr":"[fd00:1122:3344:101::c]:41614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":53815,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.637845031Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7c1b2b0b-1105-4652-9b25-72eb7a8374d0","remote_addr":"[fd00:1122:3344:101::c]:41614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.862014926Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.862175739Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a075ee49-a76d-46a1-a7e1-ff20b6d1766a","remote_addr":"[fd00:1122:3344:101::a]:39387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.962209507Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48625"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.962322739Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"849a47f4-cfc8-4792-8f0b-6b83fb86b0a7","remote_addr":"[fd00:1122:3344:101::a]:48625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:27.970769651Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0e01ab55-f18d-4df6-ad58-2048e9d13b6e","remote_addr":"[fd00:1122:3344:101::a]:48625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":775,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:28.218053997Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64183"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:28.268111987Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"05eb3b31-6043-4659-b339-f0a794ac5e56","remote_addr":"[fd00:1122:3344:101::a]:64183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49996,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:28.268886089Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"94b597c9-bbc5-432e-9d99-705c986c3838","remote_addr":"[fd00:1122:3344:101::a]:64183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":594,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:39.435589001Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"c699a4bb-caa1-44ac-933b-457304b5536a","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1227,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:51.717591143Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44949"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:51.717660484Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b35250d2-2c6f-4fdd-9709-304f2b0b4ec9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44949","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.040058072Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.040178724Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d02cfc22-fa92-474b-8f4f-b5846c50605d","remote_addr":"[fd00:1122:3344:101::c]:63218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.048255079Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"df985b18-e7ae-43b2-86b0-c62ff9bb2925","remote_addr":"[fd00:1122:3344:101::c]:63218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":140,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.116842809Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.117054552Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"55980d2e-9919-4e21-bb77-4b09e4de4855","remote_addr":"[fd00:1122:3344:101::b]:40294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.118016287Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.118112379Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b4d5cc71-6f48-4094-9d62-9911ee757448","remote_addr":"[fd00:1122:3344:101::b]:60330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.125169469Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b7ae3044-c5cc-4b56-871e-b0edda7f9517","remote_addr":"[fd00:1122:3344:101::b]:60330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":68,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.137074354Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50371"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.137205146Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f337da42-bef6-4dbb-ae3f-aba992c8bcc6","remote_addr":"[fd00:1122:3344:101::c]:50371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.206138831Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.253568801Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8c6286ef-40c7-4332-af27-51f24aef5bc6","remote_addr":"[fd00:1122:3344:101::c]:35449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47384,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.253934757Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9ea3e08a-de42-4b57-9bce-3b9d1c800573","remote_addr":"[fd00:1122:3344:101::c]:35449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.319713182Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.367822182Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d76f38f6-37e0-443c-b6cb-24cb140a2fde","remote_addr":"[fd00:1122:3344:101::b]:53090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48044,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.368779327Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a8fbcd3b-10d0-41b9-a6d7-1b867c521eb2","remote_addr":"[fd00:1122:3344:101::b]:53090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":793,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.741586441Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.742293172Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b2b3f509-a906-478b-b0d5-775c354c7c02","remote_addr":"[fd00:1122:3344:101::a]:47671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":659,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.749484064Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"631fbe2c-6e61-43f8-8c8d-9e0528b78349","remote_addr":"[fd00:1122:3344:101::a]:47671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.935216569Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:57.935371031Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"771e5624-f648-4f50-9564-11c6acb91087","remote_addr":"[fd00:1122:3344:101::a]:39338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:58.050216232Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:58.099021473Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c7cad607-8203-4e0c-ba4d-4afe029c9b21","remote_addr":"[fd00:1122:3344:101::a]:34403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48742,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:47:58.099253577Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5172c0ac-128a-4ade-98a7-eb08f9c6450c","remote_addr":"[fd00:1122:3344:101::a]:34403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:09.448823441Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"3648f403-6131-4e56-9225-3561ea3d25cc","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":303,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:21.751562953Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:21.751625954Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d3263e2a-6cef-4687-ac23-590401043668","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36958","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.147689547Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.147821579Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c1ee7b57-cc1d-4df7-9fd2-8bd80db454c3","remote_addr":"[fd00:1122:3344:101::b]:42701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.155441358Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d840455c-1cc8-41a5-84f2-8ea82f65f799","remote_addr":"[fd00:1122:3344:101::b]:42701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.173142254Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37333"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.173220425Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d2540b66-0bb8-43e9-8e3a-bd60b62edf5d","remote_addr":"[fd00:1122:3344:101::c]:37333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.180751302Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c6419742-8eed-4b4c-a1cb-aaa7a848a62a","remote_addr":"[fd00:1122:3344:101::c]:37333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.23706342Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.237189002Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"674746e7-0cf9-4a59-9522-8df36e3dbc05","remote_addr":"[fd00:1122:3344:101::b]:41444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.341275535Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.342151709Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"bc3f3ddc-63ca-4b6d-ab8b-56f97a4cb645","remote_addr":"[fd00:1122:3344:101::c]:37239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":835,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.481846307Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.527322696Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0a53a023-d116-47e0-a9c7-5eb57e4ecdfa","remote_addr":"[fd00:1122:3344:101::c]:41670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45425,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.527443428Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.527708922Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4872d067-8c9a-44e7-a4b3-41d76fb9a88d","remote_addr":"[fd00:1122:3344:101::c]:41670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.571070738Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"88aa3a26-b30f-42e3-838b-b6d9d5075f8d","remote_addr":"[fd00:1122:3344:101::b]:48107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42938,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.572088284Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2ae67c87-1c7e-4d87-834f-f8deccb2fd30","remote_addr":"[fd00:1122:3344:101::b]:48107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":766,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.9649816Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.965740742Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"24e1d6d7-f889-4031-8ece-8126939d6e97","remote_addr":"[fd00:1122:3344:101::a]:33800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":729,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.973476252Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"24025d98-96ec-45d7-aea3-4c20477b30f6","remote_addr":"[fd00:1122:3344:101::a]:33800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.987756815Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:27.987888957Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4b8af4fe-cb71-45b7-a478-5acfc26eae91","remote_addr":"[fd00:1122:3344:101::a]:64000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:28.23693222Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39374"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:28.285203613Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"577129d3-e122-4aca-9784-4abbcba496c9","remote_addr":"[fd00:1122:3344:101::a]:39374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48219,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:28.285458447Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a64766ab-b3b3-45ac-9ab9-debb213185c4","remote_addr":"[fd00:1122:3344:101::a]:39374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:39.51469863Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"8ef03903-2f6f-43ba-b402-de648a74ff16","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1305,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:51.786625733Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36420"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:51.786729335Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ad9d389a-fb0d-446a-9834-1f54bb5274f7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36420","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.032991884Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37583"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.033137256Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c0596325-180f-4c16-b075-332fa4324124","remote_addr":"[fd00:1122:3344:101::c]:37583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.039620227Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cabf868f-1c14-435e-aa33-b7b721ce1bc7","remote_addr":"[fd00:1122:3344:101::c]:37583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.053565814Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.054277626Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"72d53481-582c-4712-ab27-295770479e66","remote_addr":"[fd00:1122:3344:101::b]:61624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":676,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.061188453Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"eed70a17-0a60-4c76-95a9-e0b030d999fd","remote_addr":"[fd00:1122:3344:101::b]:61624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.068582629Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.068714181Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9e9feaba-eb88-4e1d-9c15-a80d73a682a3","remote_addr":"[fd00:1122:3344:101::b]:33362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.182401443Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.182594066Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b35b1150-2814-4e85-8120-625f0d83d69d","remote_addr":"[fd00:1122:3344:101::c]:64920","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.200088099Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.247998476Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e3ef780c-b88d-4e16-a655-75170b37198f","remote_addr":"[fd00:1122:3344:101::b]:46357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47859,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.248108428Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.251669094Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1563eaf8-f3a8-4e20-b991-755f51680248","remote_addr":"[fd00:1122:3344:101::b]:46357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3348,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.292321257Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"61c54f4f-53e4-4019-88e1-d46b7fe5396c","remote_addr":"[fd00:1122:3344:101::c]:57736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44112,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.292620192Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c0ec84e9-60cf-445b-a01a-67120777b286","remote_addr":"[fd00:1122:3344:101::c]:57736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.789282926Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.78950608Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"772f375f-8964-4082-b36a-d33e6cb3022c","remote_addr":"[fd00:1122:3344:101::a]:65468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":204,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.857810475Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.857904226Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3659d493-b6ad-4938-b522-11d8b4b79e54","remote_addr":"[fd00:1122:3344:101::a]:56861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":75,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.866716424Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4ed7cb79-2a5a-4514-96f7-edd9eec66aa0","remote_addr":"[fd00:1122:3344:101::a]:56861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:57.967643287Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:58.017571605Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e26ff743-9c02-47a1-854f-a38c2e48a173","remote_addr":"[fd00:1122:3344:101::a]:46805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49882,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:48:58.017970591Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"55e3ba24-b1d0-40e5-b5b2-c4bdad9d4c3f","remote_addr":"[fd00:1122:3344:101::a]:46805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:09.514181309Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"6b8e1a94-64aa-4ee0-b0e4-5465c2333b3a","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":644,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:21.820854684Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:21.821460303Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2f8c69bd-2b52-4b1f-b0d7-9089cb37a22f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41138","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":581,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.15161417Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.152841619Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"511fe546-05b2-4e66-9290-ab43c3b524c6","remote_addr":"[fd00:1122:3344:101::b]:43588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.184864618Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45887"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.184991371Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a5a3f8e3-98ec-46a0-8552-e594ca943f87","remote_addr":"[fd00:1122:3344:101::c]:45887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.294080261Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61882"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.294176963Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"324759bf-ea2d-47cd-b5ef-ba459fd79f60","remote_addr":"[fd00:1122:3344:101::c]:61882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.298360658Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46082"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.298655023Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5ee4aad0-04e1-4d1e-aa29-917a8eb09339","remote_addr":"[fd00:1122:3344:101::b]:46082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.302913759Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f9618bd3-65e4-491f-bb19-c332b9a60cdf","remote_addr":"[fd00:1122:3344:101::c]:61882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1032,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.306020808Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2ce90156-949e-4b18-b5f3-a3db0251ae8e","remote_addr":"[fd00:1122:3344:101::b]:46082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":54,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.529264968Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40448"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.577371779Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0443a191-4f5d-431b-bb4e-36556cf0f3bd","remote_addr":"[fd00:1122:3344:101::b]:40448","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48042,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.577538521Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.581226499Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1177c1ef-8ca5-4c70-80d5-df730a146df1","remote_addr":"[fd00:1122:3344:101::b]:40448","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3535,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.633159059Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1b312a54-3bfa-4165-b17c-f1627be9fd98","remote_addr":"[fd00:1122:3344:101::c]:52643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55545,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.633588595Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"872a13fb-ad34-404e-b027-fa35cb60a052","remote_addr":"[fd00:1122:3344:101::c]:52643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.853045257Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.853145959Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ad675a21-5f1e-4be6-8f5b-244e4ee9526e","remote_addr":"[fd00:1122:3344:101::a]:58938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:27.860034206Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0342cae0-9b05-4ce9-9294-22d6ce8fe21c","remote_addr":"[fd00:1122:3344:101::a]:58938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":154,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:28.011427026Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:28.011618509Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5361dd3c-1e3c-4a58-b19f-f851a62c9264","remote_addr":"[fd00:1122:3344:101::a]:63654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":160,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:28.247481237Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:28.29643341Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3b29d348-7b18-4273-9b84-05d107eea5f6","remote_addr":"[fd00:1122:3344:101::a]:40400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48888,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:28.296704314Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b24f97cd-5fa0-4024-b525-727625524fc6","remote_addr":"[fd00:1122:3344:101::a]:40400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:39.515018277Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"f2c1d3cc-e9f5-4746-b03d-ca41d1ea81ea","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1273,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:51.85428851Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:51.854392942Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"81727e68-497e-43d6-bb68-f651d2023220","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39450","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.073390986Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.073526618Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"37207ae3-223f-4f7d-bc73-a1dda1507c6b","remote_addr":"[fd00:1122:3344:101::b]:38040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.080767321Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f6e37050-cf70-4a03-bbaa-8aacafa040bb","remote_addr":"[fd00:1122:3344:101::b]:38040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.12308282Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41270"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.124005045Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"38235294-7291-4263-bae8-b61164a444b4","remote_addr":"[fd00:1122:3344:101::c]:41270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":883,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.188412049Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36755"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.188539171Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"02e3bc42-1220-40d6-9799-de025f888099","remote_addr":"[fd00:1122:3344:101::b]:36755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.253363522Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.254043772Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1cf7315c-04fc-4b1a-bd19-ff686a20afbb","remote_addr":"[fd00:1122:3344:101::c]:58431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":641,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.259941344Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f9a584fd-14ba-4be2-8c28-1d39d2d79aae","remote_addr":"[fd00:1122:3344:101::c]:58431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":70,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.287602726Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41394"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.337122068Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a9875548-dd0f-4d69-bf69-93542c3d473f","remote_addr":"[fd00:1122:3344:101::b]:41394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49478,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.337941851Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a3c3a8bb-7168-4b82-b0be-546a9df5a9e6","remote_addr":"[fd00:1122:3344:101::b]:41394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":685,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.338171144Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.387617985Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"36cf7a47-3acb-4306-9905-be5f9769aceb","remote_addr":"[fd00:1122:3344:101::c]:34841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49314,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.38791862Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6edf82dd-c6d8-4623-8836-5317e961eaa4","remote_addr":"[fd00:1122:3344:101::c]:34841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.837160275Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.837350698Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"39514840-e955-4eb1-9c78-1c7d1221a2a8","remote_addr":"[fd00:1122:3344:101::a]:58765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.957939077Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46026"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.958640668Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6aea9d98-4889-42fe-9f44-d43088cbf02e","remote_addr":"[fd00:1122:3344:101::a]:46026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":679,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:57.966810365Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ae0c32e9-d6c3-48f0-a7a3-420da8398ea5","remote_addr":"[fd00:1122:3344:101::a]:46026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:58.080665801Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52024"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:58.126255802Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"069d4726-0597-4f88-a4fb-3b33f4e03eec","remote_addr":"[fd00:1122:3344:101::a]:52024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45542,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:49:58.127156385Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"22a86915-c42c-4ac0-ab69-88500d871467","remote_addr":"[fd00:1122:3344:101::a]:52024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":746,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:09.515341619Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"c775d032-981f-417b-8cd7-324a30a64a5e","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":929,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:21.886526239Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:21.88658468Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f389cf45-a52c-4c59-821e-ef64cc5e4ef4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55761","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.173407152Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.173616285Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1467d148-a69e-411e-a130-dc7030ed2b65","remote_addr":"[fd00:1122:3344:101::b]:32814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.217685692Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.217816364Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a3b4e934-4534-4f85-85d4-66ef2288f817","remote_addr":"[fd00:1122:3344:101::c]:38306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.225383632Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2f40d144-d720-429c-8025-0df10b76298e","remote_addr":"[fd00:1122:3344:101::c]:38306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.310906916Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48279"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.310972107Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"895fb1c8-e0d1-4f5e-a06e-c513bda77784","remote_addr":"[fd00:1122:3344:101::b]:48279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.319131494Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a1736d09-9cf3-4b9f-b71f-471affaa3c6e","remote_addr":"[fd00:1122:3344:101::b]:48279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.321382889Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50027"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.322377655Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"18820ee5-f8c2-4f67-863e-2882c747943e","remote_addr":"[fd00:1122:3344:101::c]:50027","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":969,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328320727Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61707"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328565581Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328604632Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328630502Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328648952Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328680353Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328691013Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328704343Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.328720014Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e737c558-f7c9-489b-ba21-0101fd4df9a0","remote_addr":"[fd00:1122:3344:101::b]:61707","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":334,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341736496Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44565"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341856018Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341876989Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341897899Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341907549Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.34192241Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.341928839Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.34194021Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.34195314Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f212c7b8-7159-4152-88c3-ec880d55da79","remote_addr":"[fd00:1122:3344:101::c]:44565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.482930788Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.530644722Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"41f48121-bb9c-4464-9c3c-b91c5d6c2e19","remote_addr":"[fd00:1122:3344:101::c]:53160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47658,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.530775944Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.534536813Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"46623190-e292-470d-82c2-e504c635372c","remote_addr":"[fd00:1122:3344:101::c]:53160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3487,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.576607029Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9cd3f1db-a6e3-4eaa-93d9-78d950408b3d","remote_addr":"[fd00:1122:3344:101::b]:47075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45734,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.576993755Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"66d40e09-7f8b-4a82-86b3-6e73e841ae3a","remote_addr":"[fd00:1122:3344:101::b]:47075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.84795611Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.848060391Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5b8f9bd5-9755-4131-b4bb-b3d35680a17d","remote_addr":"[fd00:1122:3344:101::a]:64875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:27.855231563Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9523eea1-85f1-4b62-ab0d-e17047bb5ef0","remote_addr":"[fd00:1122:3344:101::a]:64875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.071339182Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65326"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.071491714Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5b0a1333-5997-41c6-bc93-e70798ac2446","remote_addr":"[fd00:1122:3344:101::a]:65326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133465961Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33341"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133678104Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 217faa24-06af-441e-b2c5-d0c1ff63e684, kind: Service { id: 94a2c9b0-295c-488e-99e0-bfb110e44e3a }, name: Name(\"external-dns-94a2c9b0-295c-488e-99e0-bfb110e44e3a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 147])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf874f69-f7ba-4d95-8dd9-e495e2d30c0d, kind: Service { id: 8b280afb-d117-4f80-a3f2-2835451a0bc9 }, name: Name(\"external-dns-8b280afb-d117-4f80-a3f2-2835451a0bc9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 385a3868-0e09-45a7-a6e5-296ee3691fc5, kind: Service { id: 3662ecea-b03d-4e92-9b8c-8abcae1826e2 }, name: Name(\"nexus-3662ecea-b03d-4e92-9b8c-8abcae1826e2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 60])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b009bd3e-45e6-40b1-b56d-25a386c080a8, kind: Service { id: 5da8cc16-12a1-4dfc-8dc7-6845077fc9c5 }, name: Name(\"nexus-5da8cc16-12a1-4dfc-8dc7-6845077fc9c5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 150])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: dbb1a2f5-f836-4e9a-a3cb-f43b48fee365, kind: Service { id: 1d5b81a3-381f-4d1b-b975-02e6284ad71f }, name: Name(\"nexus-1d5b81a3-381f-4d1b-b975-02e6284ad71f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 167])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133715215Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133748295Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133767076Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133780866Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133797576Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133813846Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.133827866Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c6dd696e-42f0-4aa5-94f0-e6defacb540f","remote_addr":"[fd00:1122:3344:101::a]:33341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":324,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.342031143Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34028"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.389247939Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8b43190d-7401-4572-a2b6-ba1fbb40da50","remote_addr":"[fd00:1122:3344:101::a]:34028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47120,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:28.389797858Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b7be35fe-731e-4acd-b24e-2f627446f879","remote_addr":"[fd00:1122:3344:101::a]:34028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":302,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:39.515243745Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"aa7ba471-cfdb-43e3-9208-bc23e27b75e1","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":959,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:51.919144443Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:51.919244475Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2e89eb99-654d-4d64-bff5-61f00143944b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41835","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.079913979Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.080050562Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b42ac12a-6196-4d89-9678-ef54535b5278","remote_addr":"[fd00:1122:3344:101::b]:46422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.083519096Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:59009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.083592747Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"628b21d5-b39a-442a-8304-3ff0df400410","remote_addr":"[fd00:1122:3344:101::c]:59009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.088061806Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"727c644c-ec77-46a4-97db-d6f3b772afff","remote_addr":"[fd00:1122:3344:101::b]:46422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.090553095Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b625a46f-59d4-4695-bf8f-2e0395cb3641","remote_addr":"[fd00:1122:3344:101::c]:59009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.189166423Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.189333555Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"776e8fb0-8468-4fba-a0be-76c7dbbf8b0f","remote_addr":"[fd00:1122:3344:101::c]:39807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.192885641Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.193017903Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"25715724-04ea-47a5-9f6b-1f610ad7c7f3","remote_addr":"[fd00:1122:3344:101::b]:45477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.282987486Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.330647519Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"dcc5f4c1-a232-4864-a594-ff6c7156b942","remote_addr":"[fd00:1122:3344:101::c]:39843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47604,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.330766411Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.334470189Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aedb811b-90e0-4085-bb82-564383b34505","remote_addr":"[fd00:1122:3344:101::c]:39843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3476,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.377439479Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"22b11e09-9a8e-4c87-9049-df2e28270250","remote_addr":"[fd00:1122:3344:101::b]:42100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46549,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.378000817Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b18a133a-dbe1-4b9d-b08c-ac7f2c831ba9","remote_addr":"[fd00:1122:3344:101::b]:42100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.761875313Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34501"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.761997725Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"82892ee6-414e-4f68-8551-6915935fe65e","remote_addr":"[fd00:1122:3344:101::a]:34501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.769307409Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c05d702e-dd4f-49f0-98ca-bcc01750dfdd","remote_addr":"[fd00:1122:3344:101::a]:34501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.905800666Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:57.905963799Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d0e0214d-1e9a-495a-8e8e-07aa3fbac897","remote_addr":"[fd00:1122:3344:101::a]:61664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.011486154Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:32941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.05545382Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7f84cb38-80f4-4df6-9791-5703093fdd9d","remote_addr":"[fd00:1122:3344:101::a]:32941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43900,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.055730974Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d8ea9cef-d582-4d1d-9581-b490bff339f2","remote_addr":"[fd00:1122:3344:101::a]:32941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.964116867Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53788"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.964358641Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2f4ea11f-9358-4967-ae75-81e803b1860c","remote_addr":"[fd00:1122:3344:101::a]:53788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":196,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.964617625Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11830577) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:58.964661436Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"e54cad2d-98dd-4085-a3b1-454f8d225d57","remote_addr":"[fd00:1122:3344:101::a]:53788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.098232838Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40699"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.111086249Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:577","metadata":"InstanceMetadata { silo_id: 38c0ddb2-1802-4247-8997-e553c8ee958c, project_id: b4ce4c14-fa8a-4e52-bff9-09f6e27282e2 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-19T18:50:58.720011Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11830577))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6d3cf39e544\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, read_only_parent: Some(Volume { block_size: 512, id: b324db2d-87a4-43a6-822d-3ffd9d432d72, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 582a6bc9-b5fc-4fc9-b8d3-f2ea28a5485a, key: Some(\"E417JzeZiNSBozUCA/zaVeo2ld3lfdV7qwsC//lQP2c=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, key: Some(\"1Zj4iZh8U9bZNQI6+KiJJmjBzyfavzunSwPraiwHp20=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABpXZbzWPNYAABdlvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAal2W81jzWAAAXZbzWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmNGY0ODFjOC1kMTI5LTRmYzUtOTljZS1jNmM1M2ZlYTdkOWUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDUrNmEvODd1a3NTUDB5eWRuWWtMeVlUbUgwUVMyUmR2VG0wcFJDRS9LNCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.111196561Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:611","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.11178369Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance.rs:951","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11830577))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9c77785d-3a87-413d-90db-56b1fd08e90d, kind: Instance { id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11830577), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6d3cf39e544\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, read_only_parent: Some(Volume { block_size: 512, id: b324db2d-87a4-43a6-822d-3ffd9d432d72, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 582a6bc9-b5fc-4fc9-b8d3-f2ea28a5485a, key: Some(\"E417JzeZiNSBozUCA/zaVeo2ld3lfdV7qwsC//lQP2c=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, key: Some(\"1Zj4iZh8U9bZNQI6+KiJJmjBzyfavzunSwPraiwHp20=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABpXZbzWPNYAABdlvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAal2W81jzWAAAXZbzWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmNGY0ODFjOC1kMTI5LTRmYzUtOTljZS1jNmM1M2ZlYTdkOWUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDUrNmEvODd1a3NTUDB5eWRuWWtMeVlUbUgwUVMyUmR2VG0wcFJDRS9LNCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-19T18:50:58.720011Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.112623423Z","hostname":"buskin","pid":1562,"uri":"/instances/f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","method":"PUT","req_id":"f7003061-deb1-4011-ba38-860bdc7fa20e","remote_addr":"[fd00:1122:3344:101::a]:40699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":14346,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.24779039Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47535"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.249959274Z","hostname":"buskin","pid":1562,"uri":"/instances/f4f481c8-d129-4fc5-99ce-c6c53fea7d9e/state","method":"PUT","req_id":"85399c89-f5dc-4c43-9efe-732d57ab3583","remote_addr":"[fd00:1122:3344:101::a]:47535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2099,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.251963426Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 101, 110]), slot: 0, vni: Vni { inner: 11830577 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.252012726Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.287764174Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:50:59.310531899Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:00.583886613Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:00.584106956Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 28.352885ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:00.773773524Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 58.235133ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:00.810981974Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 244.51276ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:00.886993368Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 446.874373ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:01.14746181Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 899.498523ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:01.607210869Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 892.767124ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:02.518168762Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 1.117176564s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:03.420961398Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 1.141327844s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:04.550006772Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:05.720034395Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1460"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 63.220999ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:05.728625049Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 73.838713ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:05.802740905Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 196.411203ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:05.88716623Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\") failed: Property not found. retry in 272.247527ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:06.093387346Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.376893446Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1469"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.411593467Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1483"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABpXZbzWPNYAABdlvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAal2W81jzWAAAXZbzWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmNGY0ODFjOC1kMTI5LTRmYzUtOTljZS1jNmM1M2ZlYTdkOWUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDUrNmEvODd1a3NTUDB5eWRuWWtMeVlUbUgwUVMyUmR2VG0wcFJDRS9LNCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-6d3cf39e544\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, read_only_parent: Some(Volume { block_size: 512, id: b324db2d-87a4-43a6-822d-3ffd9d432d72, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 582a6bc9-b5fc-4fc9-b8d3-f2ea28a5485a, key: Some(\"E417JzeZiNSBozUCA/zaVeo2ld3lfdV7qwsC//lQP2c=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 59531927-9116-4d2b-a646-fdfc19e8b1f3, key: Some(\"1Zj4iZh8U9bZNQI6+KiJJmjBzyfavzunSwPraiwHp20=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: f4f481c8-d129-4fc5-99ce-c6c53fea7d9e, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: b4ce4c14-fa8a-4e52-bff9-09f6e27282e2, silo_id: 38c0ddb2-1802-4247-8997-e553c8ee958c }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.415494178Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:698"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.584561984Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:700"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-19T18:51:06.585596591Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.585617751Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.586096568Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-19T18:51:06.585596591Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.586108379Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }, propolis_id: 8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-19T18:51:06.585596591Z }, migration_state: None }","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-19T18:51:06.612592352Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.612606012Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.612618772Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-19T18:51:06.612592352Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.612625522Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }, propolis_id: 8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-19T18:51:06.612592352Z }, migration_state: None }","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.650770137Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.652189939Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7213f32f-2d82-41e1-a653-0b72d4093c9a","remote_addr":"[fd00:1122:3344:101::a]:47648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1366,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.697047318Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"98a1aa9d-ee34-47d0-9eb1-89718d5d746c","remote_addr":"[fd00:1122:3344:101::a]:47648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":264,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.740277423Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.740415975Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ae7241d5-f737-48e4-b1fd-274a9ba38031","remote_addr":"[fd00:1122:3344:101::a]:63584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:06.752043116Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1433e76d-1a33-44a0-ab91-7791f5f3fcd5","remote_addr":"[fd00:1122:3344:101::a]:63584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:09.598950814Z","hostname":"buskin","pid":1562,"uri":"/447ca897-407e-4dad-a0e2-4f6036da9497","method":"GET","req_id":"4b90c266-18ea-4d47-8ff1-71b65603748f","remote_addr":"[fd00:1122:3344:101::d]:47370","local_addr":"[fd00:1122:3344:101::1]:35720","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"447ca897-407e-4dad-a0e2-4f6036da9497","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":604,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.665412983Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.666175665Z","hostname":"buskin","pid":1562,"uri":"/instances/f4f481c8-d129-4fc5-99ce-c6c53fea7d9e/state","method":"PUT","req_id":"5a70a92a-0729-4fb2-aba6-eae3897bcec8","remote_addr":"[fd00:1122:3344:101::a]:42922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":688,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-19T18:51:16.666336161Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.666364689Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.666389918Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-19T18:51:16.666336161Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.666436371Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T18:50:58.744812839Z }, propolis_id: 8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-19T18:51:16.666336161Z }, migration_state: None }","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-19T18:51:16.686282799Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.686308269Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.686329169Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-19T18:51:16.686282799Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T18:51:16.686282799Z }","propolis_id":"8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.68634286Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:621","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.68635008Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.688881069Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.6889122Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:445","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/92e7db57-0bba-4f49-937c-e777282668a4/debug/bundle/zone\", \"/pool/int/b9419da7-be18-4d81-955a-2078bb5c8da4/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/9e607437-7489-43a3-8665-a0998816c2da/crypt/debug\", \"/pool/ext/9792c5a6-e0f4-4250-91d8-71386ac84474/crypt/debug\", \"/pool/ext/61f4fa9a-1baa-4a1b-8076-77046f3a1be7/crypt/debug\", \"/pool/ext/3481a5aa-d464-43f4-8c3b-e7c18df1676b/crypt/debug\", \"/pool/ext/6d07c0c6-0259-4eb0-8ccb-3b66cdcf2293/crypt/debug\", \"/pool/ext/ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crypt/debug\", \"/pool/ext/2db534d5-2c49-4cc7-bcdd-c78bfddad2ae/crypt/debug\", \"/pool/ext/56328015-6e5b-46f4-8b73-bfdc206d32b1/crypt/debug\", \"/pool/ext/59a6abe1-72b0-47e0-8dd6-231a2ecca4d7/crypt/debug\"] }","zone_name":"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.72293605Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.72674971Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"d64a1c24-5cd9-4239-b26f-a8f9a52d8084","remote_addr":"[fd00:1122:3344:101::a]:63341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3692,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:16.739197234Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"88e4169f-289d-4af2-909b-5c80c2d4d7d9","remote_addr":"[fd00:1122:3344:101::a]:63341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":189,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:17.312079825Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1246","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc\", bundle_id: 99b78c82-a67a-42a9-9a12-65c73b54944b }, time_created: 2024-07-19T18:51:16.689150753Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T18:51:17.312166947Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:793"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.058991473Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.077995589Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(11830577), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.07804997Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(11830577), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.078464186Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T18:51:16.686282799Z }, propolis_id: 8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-19T18:51:19.078429956Z }, migration_state: None }","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.19924632Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.200167544Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f9af09aa-8cca-4ea1-a556-42fb9805b3d6","remote_addr":"[fd00:1122:3344:101::a]:49457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":873,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.200413088Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11830577) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.200454719Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"7e84610c-a479-4e37-993d-94c0e749566e","remote_addr":"[fd00:1122:3344:101::a]:49457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.349169238Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T18:51:16.686282799Z }, propolis_id: 8f266da2-e8e3-4c2b-b6f5-a8e9b5c97bbc (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-19T18:51:19.078429956Z }, migration_state: None }","instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.35317311Z","hostname":"buskin","pid":1562,"instance_id":"f4f481c8-d129-4fc5-99ce-c6c53fea7d9e","component":"InstanceManager","file":"sled-agent/src/instance.rs:736"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.404894336Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.404983128Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b25c9b04-8fe9-4cd2-9ed2-311e494617ee","remote_addr":"[fd00:1122:3344:101::a]:40107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.411875365Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"fadbf7e8-e8fc-4588-8e08-7705555e9195","remote_addr":"[fd00:1122:3344:101::a]:40107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":750,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.457294923Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.457434886Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"2c18a054-c28a-4cfd-a080-07052f0879cd","remote_addr":"[fd00:1122:3344:101::a]:56493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.463801665Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e3facad7-6312-4936-b217-8e37b68e9794","remote_addr":"[fd00:1122:3344:101::a]:56493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.574965098Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.575138781Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a127b6d3-8a48-4d0a-9bd5-beab1d67ed64","remote_addr":"[fd00:1122:3344:101::a]:50875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":149,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.575362174Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:820","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 101, 110])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11830577) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:19.575388845Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"DELETE","req_id":"ee136b9e-9b29-48ae-912e-85cf5a713751","remote_addr":"[fd00:1122:3344:101::a]:50875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":91,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:20.419168081Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40910"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:20.419252562Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"844b26df-747d-493d-ba9e-ac0d544e2827","remote_addr":"[fd00:1122:3344:101::a]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:20.425116024Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4b22073c-b504-457e-805b-9e3543b977ae","remote_addr":"[fd00:1122:3344:101::a]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:21.952844023Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T18:51:21.952940725Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0dcc65fe-0acf-423b-8e6d-42dd2dcab8f2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50931","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"}