[ Oct 24 18:51:13 Enabled. ] [ Oct 24 18:51:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 24 18:51:13 Method "start" exited with status 0. ] [ Oct 24 18:51:13 Rereading configuration. ] [ Oct 24 18:51:13 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.241121447Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.241270839Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.29386595Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.29390729Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.408365935Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.417526736Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:13.425020021Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.024590623Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163334432Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163420993Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163561515Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163660637Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163697967Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.163703817Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166662503Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:14.166669273Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166685563Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166696723Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166728214Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166772035Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166862276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166883006Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.166894737Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.177381478Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.177405259Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"1efc7c65-a878-483c-82d1-872a03292008 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.295993657Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.802808389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.80285113Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.80285769Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.80286842Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.80288025Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.80289989Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.802904301Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.810919294Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.810942625Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"982c7515-f8a4-4d54-b007-05451b94e502 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:14.919469447Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452639606Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452690647Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452730298Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452737088Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452755748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452791199Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452823489Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452831819Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452845869Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452850329Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.45287252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.45288836Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452921891Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452926881Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452947141Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452952631Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452956541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452961891Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452971491Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452983802Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.452999502Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453012912Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453017242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453021602Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453026602Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453030382Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453046772Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453052543Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453056373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453061373Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453065423Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453071873Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453095513Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453100233Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453107073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453110633Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453114444Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453126744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453130674Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453158164Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453163274Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453167334Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453170614Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453180345Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453184935Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453218395Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453223105Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453227575Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453230775Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453234575Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453260706Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453265166Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453269216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453275306Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453279286Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453288726Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453292096Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453321837Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453326387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453329867Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453333387Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453395288Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453409678Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453418108Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453430018Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453436899Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453441019Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453469289Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.453473359Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464203824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464230975Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999923139s","next_cleanup":"Instant { tv_sec: 3319, tv_nsec: 523085444 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464299756Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464308516Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464370657Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.464386387Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.467409184Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469155331Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469589928Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469599028Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469665659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.46972062Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.46973336Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.4697388Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1136"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.46976776Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469778561Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469785301Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469789501Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1178"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469793041Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469796691Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469823031Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.469855092Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470196987Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: d6d74c37-fa13-4dad-8a2e-066574b53674,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 8ed5384f-0099-459e-a18a-fdd93abd3452 (sled),\n rack_id: d6d74c37-fa13-4dad-8a2e-066574b53674,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470479391Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470486601Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470534742Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470548872Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470717085Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470854147Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.470905288Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8ed5384f-0099-459e-a18a-fdd93abd3452 (sled), rack_id: d6d74c37-fa13-4dad-8a2e-066574b53674, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.471134902Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.471195172Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.475830304Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.481522322Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.481632183Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.485707456Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.485767477Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.485860088Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:55165","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8ed5384f-0099-459e-a18a-fdd93abd3452 (sled), rack_id: d6d74c37-fa13-4dad-8a2e-066574b53674, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.48597953Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.485994571Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486040581Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486045941Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486066792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486073422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486106052Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486113622Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486142233Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486156153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486160473Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486164663Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.486233444Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.488724673Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.501948566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.521862653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.539079959Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.573375487Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:401"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.573420358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.573425548Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:413"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.573429898Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 62.082328ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.576732179Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8040fe1-11b1-4cb2-82d6-f5801ba2b291\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8040fe1-11b1-4cb2-82d6-f5801ba2b291\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_1efc7c65-a878-483c-82d1-872a03292008/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.586602481Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_1efc7c65-a878-483c-82d1-872a03292008/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.619335726Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_1efc7c65-a878-483c-82d1-872a03292008/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.619388127Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.624332943Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:428"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.624341133Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_1efc7c65-a878-483c-82d1-872a03292008/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.624345363Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.639801362Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 133.514237ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.64033053Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"303c7676-48c3-4fcd-84d8-e8d666277d80\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"303c7676-48c3-4fcd-84d8-e8d666277d80\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.717257355Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.775375591Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 109.359491ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.775967Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70062515-b3f1-427c-bb7d-d66ff1122e13\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"70062515-b3f1-427c-bb7d-d66ff1122e13\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.88623522Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 223.715756ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.886550015Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13cbf6b8-9e1e-4c0d-890c-5ec914be6038\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13cbf6b8-9e1e-4c0d-890c-5ec914be6038\" }"} {"msg":"Mounting oxi_1efc7c65-a878-483c-82d1-872a03292008/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.936666118Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.949342983Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.949438484Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.949469475Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:15.949502596Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.111562843Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 111.567716ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.112354826Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8cbc4580-1148-4175-a928-54ccd2a15506\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8cbc4580-1148-4175-a928-54ccd2a15506\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.225431849Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 211.037209ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.226100209Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45e6877a-9136-4929-9573-26f511cceec9\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45e6877a-9136-4929-9573-26f511cceec9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.438211139Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 373.733654ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.438686476Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92b0c8dd-5482-451c-b575-650980dc8ffc\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92b0c8dd-5482-451c-b575-650980dc8ffc\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.697966703Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.703376856Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.703415907Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.703470648Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:62798","producer_id":"8ed5384f-0099-459e-a18a-fdd93abd3452"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:16.704720037Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"162.624532ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.7055714Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.7055788Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.705641391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.705646761Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:16.81262996Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:16.868987149Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"516.826234ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:17.387829126Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"853.3423ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:18.242784724Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.800611652s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.050777209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2679"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.05080226Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2897"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.050808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2549"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.05082581Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2596"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.050866321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2978"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.050870791Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.05276589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2822"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.054353655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.054471846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:19.054601988Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 66.726398ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:19.323844369Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 110.323987ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:19.515393702Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 218.241681ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:19.68978873Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 260.72354ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:20.019675514Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 818.223688ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:20.384214674Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:21.044940259Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.225454386s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.136278218s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:21.371277669Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.392131418s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:22.76228274Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:23.272358433Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.049056898s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.086381142s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:24.331425168Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.913998283Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914060924Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914067174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914087754Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914319528Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914562842Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914621663Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914633053Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:25.914652463Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:55165","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130287326Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130349437Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130410308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130450599Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130460629Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.130473769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.13049719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.13050923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.134847026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4245","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.134864587Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4261"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.134872617Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.392264095Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:42502"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.392546529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.392557519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.39263323Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.392648691Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.392746752Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"a5631254-dfbf-4b12-8ab4-cd3bb2b7f696","remote_addr":"[fd00:1122:3344:101::1]:42502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":321,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.422546141Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:45055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.422745675Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"18224df6-45ae-4c37-86fc-34d45326c265","remote_addr":"[fd00:1122:3344:101::1]:45055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.424113696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.424149516Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.424490191Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.424718125Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:882"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 300c5063-3843-4f69-8229-9ee1b491e4cc, pool_id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 94f2a8f9-b7b0-4faf-b94d-03d405f7f4f7, pool_id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1d5797d0-06c1-44f3-9970-7b698b784a74, pool_id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 615fa03e-acd5-4baf-a6d7-a8074bda00a1, pool_id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 1d494720-f8b3-4800-83b0-167353ac7349, pool_id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: df9e94c4-80dd-47ab-a700-4cf718e8f0a6, pool_id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: c3e67ba2-dedc-4fd0-98a5-f5a5dcebe0e1, pool_id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: c13cc26b-0091-42f0-8d2d-ce37a90416c7, pool_id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c8b19bd7-8b6b-430a-9ad7-7e7bf6acba56, pool_id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.456902881Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457031923Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41886"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457254216Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:874"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 300c5063-3843-4f69-8229-9ee1b491e4cc, pool_id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 94f2a8f9-b7b0-4faf-b94d-03d405f7f4f7, pool_id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1d5797d0-06c1-44f3-9970-7b698b784a74, pool_id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 615fa03e-acd5-4baf-a6d7-a8074bda00a1, pool_id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 1d494720-f8b3-4800-83b0-167353ac7349, pool_id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: df9e94c4-80dd-47ab-a700-4cf718e8f0a6, pool_id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: c3e67ba2-dedc-4fd0-98a5-f5a5dcebe0e1, pool_id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: c13cc26b-0091-42f0-8d2d-ce37a90416c7, pool_id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c8b19bd7-8b6b-430a-9ad7-7e7bf6acba56, pool_id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457289217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457304927Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457319117Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457323608Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457336888Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457340998Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457345748Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.457349538Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.60735845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.607387551Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.818144999Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:26.819640943Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5874841116940550694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:26.819769845Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12271670144371597784"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:26.916175071Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.936054267Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.9446852Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944772982Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944780562Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944798512Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944809682Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944919944Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.944931214Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.989519651Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a5a77acc-8f53-4c36-a482-41fc0a50e4bd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:26.996920985Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_a5a77acc-8f53-4c36-a482-41fc0a50e4bd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.214580621Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.222142987Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.222171758Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.222193888Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.222200078Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.222210938Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.2314229Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.231450141Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.344409562Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.351203987Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.351244897Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.351250597Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.351277058Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.351284388Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.35142173Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.3514278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.392475173Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_915c0c40-3159-4259-9f5f-ef7ec918e13c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.399160636Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_915c0c40-3159-4259-9f5f-ef7ec918e13c","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.620776412Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.62843832Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.628468381Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.628474691Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.628481121Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.628490671Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.638415514Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.638439944Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7c590c44-e234-44db-965e-9f6e7745f594 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.757907306Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.765924929Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.76597868Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.76599597Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.766030241Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.766049591Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.766174063Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.766181193Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.811702165Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7c590c44-e234-44db-965e-9f6e7745f594/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:27.819386084Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:27.918050345Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_7c590c44-e234-44db-965e-9f6e7745f594","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.042449702Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.050294283Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.050338904Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.050344884Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.050351844Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.050373274Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.06045579Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.06048082Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bb91d72d-293b-493d-a375-9a8656905e43 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.18503014Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193301717Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193340198Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193349218Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193427029Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.19343853Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193588942Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.193600852Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.243192517Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bb91d72d-293b-493d-a375-9a8656905e43/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.250792724Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_bb91d72d-293b-493d-a375-9a8656905e43","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.467471474Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.47501846Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.475068741Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.475076091Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.475082471Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.475089101Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.485201527Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.485237218Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"fa33182c-5386-4788-bb69-fed5eea70499 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.601478249Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609015806Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609050516Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609059016Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609080707Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609109687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609246489Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.609253569Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.650917931Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fa33182c-5386-4788-bb69-fed5eea70499/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.658799783Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_fa33182c-5386-4788-bb69-fed5eea70499","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.879703118Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.889179564Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.889216925Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.889222745Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.889229425Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.889235605Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.898406687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:28.898438427Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.017639224Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025186801Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025223441Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025231682Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025239282Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025249832Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025356454Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.025369444Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:29.028780266Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.072924247Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.081159534Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.310721853Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.319060711Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.319168063Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.319183533Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.319192493Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.319199593Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.330469877Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.330509938Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.456911066Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465058672Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465118142Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465126443Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465146183Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465155523Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465281785Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.465288665Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.513700031Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ebb69d76-d0be-447a-903a-9633ca2c2c1a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.521599813Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_ebb69d76-d0be-447a-903a-9633ca2c2c1a","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.744106123Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.75235234Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.752395391Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.752410221Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.752415621Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.752419751Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.762586628Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.762609548Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.890043483Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898078736Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898118407Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898124547Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898129947Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898138857Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.898274459Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.89828139Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.942960288Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:29.951387798Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:30.031189668Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.175135117Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.18306729Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.18310498Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.18312105Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.183128351Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.183135561Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.192968892Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.192995463Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.311761024Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.319803148Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.319852098Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.319908209Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.31997603Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.31998914Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.320145613Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.320154383Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:30.323809219Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.976950145s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.368520948Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6063ad03-661e-4d00-b0bf-ebbe587f7c8b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.376493901Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_6063ad03-661e-4d00-b0bf-ebbe587f7c8b","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.600936561Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.608940845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.608979855Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609063546Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609196309Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1efc7c65-a878-483c-82d1-872a03292008 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 982c7515-f8a4-4d54-b007-05451b94e502 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609256189Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609263039Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609380711Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609656345Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:878"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609691636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609714846Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609721006Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:909"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609727377Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:914"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609775777Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609784567Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:920"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609851688Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609863419Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.609972541Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"ce9a20a7-dbdd-4e00-8038-3265021b6007","remote_addr":"[fd00:1122:3344:101::1]:41886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4152852,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.640973698Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.641123791Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39345"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.641337224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.641343564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.707557535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.707611495Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.770259651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.770300702Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.834920798Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.834973679Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.834984899Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.83503513Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.880737644Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.880777085Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.880783385Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.932726556Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.936330611Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.939856026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.952171465Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.956555643Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:30.960672867Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:31.03289302Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:32.034406658Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:32.562732572Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:32.562785783Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:32.563033547Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.04722406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.528995616Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.529111558Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 42.196362ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.536499652Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.536561073Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.536641244Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:33.541166274Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 141.505456ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.588862859Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 41.404705ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.666147051Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 30.798677ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.672281985Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 55.049349ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.710342802Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 127.618978ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.716290924Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 109.564965ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.821071199Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 233.573836ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.964072073Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 254.309947ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:33.99829136Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 229.794005ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.017820252Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 225.096981ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.209849532Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 283.441264ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.258680005Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 876.997241ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.267782195Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 882.848135ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.444121773Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 885.397904ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:34.550519133Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 1.07192981s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.153530008Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 766.607568ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.337015617Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.418219319Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4269"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.418274309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4279","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.425334838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4358"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 918.213315ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.447237206Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.549085806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4406"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.549132246Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.594646278Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.594689399Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4310"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.658085526Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4352"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.658126407Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4413","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.680699525Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4424"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.680739425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.760722038Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4498"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.760764249Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4504"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.866252945Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:35.866308266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ed5384f-0099-459e-a18a-fdd93abd3452)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.899229833Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Failed to get switch ID from MGS (retrying in 42.928112ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.899321585Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:35.915977761Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13353030731136285950"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:35.916088393Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6387999463950500103"} {"msg":"Failed to get switch ID from MGS (retrying in 104.914127ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:35.943240022Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 207.518707ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:36.049490559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 1.038210762s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:36.113248952Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 1.496127203s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:36.234186056Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.247217677Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:64894"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.247542172Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"7f2028b5-b5d4-48ab-bd8c-cf8c75a61830","remote_addr":"[fd00:1122:3344:101::2]:64894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.259489366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.291127574Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.291231865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.291796824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.291829945Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.327469204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374219264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374279695Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374292316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374303506Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374310926Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374319316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374326416Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374333026Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374340806Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374347337Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374353666Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374360117Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374366817Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374390617Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374399597Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374404247Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374408757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374415167Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374421878Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374426678Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374431398Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374437858Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:36.374442338Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 1.27989194s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:36.374447948Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:36.822005257Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5874841116940550694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:36.822131079Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12271670144371597784"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 751.429609ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:37.161144335Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 640.500483ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:37.664054227Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d\") failed: Property not found. retry in 986.522663ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:37.737954676Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29\") failed: Property not found. retry in 1.072437806s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:37.922044654Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19\") failed: Property not found. retry in 514.481976ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:38.313468478Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016016378Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_06542437-1810-4294-a723-9ed308666b19"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016065788Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_d4b09213-5961-4095-a130-f1ddc420cb29"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016071979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_26818eb7-96aa-486e-ac98-9a5f4e70076d"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016084199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016401893Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.016695988Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2d60b681-34c1-4a2e-b8a0-b2b1e3543513","remote_addr":"[fd00:1122:3344:101::1]:39345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8375503,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.01941126Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"failed to write DNS configuration (will retry in 62.624189ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:39.138245662Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:595","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"failed to write DNS configuration (will retry in 80.32734ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:39.202444981Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:595","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"failed to write DNS configuration (will retry in 119.080612ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:39.284683089Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:595","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.443611759Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"failed to write DNS configuration (will retry in 84.444978ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:39.473411678Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:595","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.584635183Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.637747942Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.637766742Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.668247702Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.669130405Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.669147375Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.697808997Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.697897699Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37708"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.698074192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.698081021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.719754876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.719804476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.738903061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.738935551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.757382946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.757421486Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.757565578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.75764198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.768780322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.800544911Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.80114523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.801151061Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.83031142Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 219, 3]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.83032238Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.830659185Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.830668516Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.830928749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.831390707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.854407681Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:39.871941432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:40.511433358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:40.51150519Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 61.581293ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:40.643748628Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 75.533814ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:40.723947844Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 265.218239ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:40.839753299Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 277.21161ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:41.121778037Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 714.743847ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:41.408614708Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 540.893388ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:42.134053421Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 1.006690493s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:42.684939403Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 1.337571932s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:43.701121146Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544\") failed: Property not found. retry in 983.028931ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:45.048242911Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:45.91811328Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13353030731136285950"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:45.918229272Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6387999463950500103"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.051803171Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.051866251Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.052223037Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.052543712Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2ceeab8f-874d-41a1-8345-e405f9f99572","remote_addr":"[fd00:1122:3344:101::1]:37708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6354597,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.054981369Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.084018047Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.084148659Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55929"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:46.092539948Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.0926331Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"c71e29e8-1c41-4e9a-8ad2-1ad8e1c46b3f","remote_addr":"[fd00:1122:3344:101::1]:55929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8344,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:46.09520832Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.27102073Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:46.271238453Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60639"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:46.824135646Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5874841116940550694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:46.824257228Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12271670144371597784"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:48.748945116Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7389690465290371847"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:48.852605824Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14791933361063627293"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:49.018909277Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18179449242114240319"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:50.279494489Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.279627251Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"ba3ce4da-c9f7-4dec-a8eb-a0d23d8ea535","remote_addr":"[fd00:1122:3344:101::1]:60639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008303,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:50.282622747Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.440739565Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.440890577Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36489"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:50.449761454Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.449829075Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"8f0ce286-ea43-4296-8502-74af0538069f","remote_addr":"[fd00:1122:3344:101::1]:36489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8878,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:50.453103915Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.685724151Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.685891084Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53658"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:50.693701644Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.693810126Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"9902899b-38aa-46af-b044-66930b2787ec","remote_addr":"[fd00:1122:3344:101::1]:53658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7847,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:50.696349705Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.962719851Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:50.962889354Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:49725"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_638a0656-b644-4d47-9872-6e9a903f4544': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-24T18:51:51.971673293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:51.971754774Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"3c660493-4504-4dc5-8c79-d9f51fb6eec1","remote_addr":"[fd00:1122:3344:101::1]:49725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":1008803,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:51.97405377Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.214894102Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.215043565Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:46169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.223310122Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"f9116a0c-94ff-473a-ad47-c32dfb88ea68","remote_addr":"[fd00:1122:3344:101::1]:46169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8219,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.226093975Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:52.226104365Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.43982519Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.439954382Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:47649"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.44827674Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"68b11a93-606b-4740-a036-a7a0dc0b7456","remote_addr":"[fd00:1122:3344:101::1]:47649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8269,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.449911565Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:52.449924356Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.794860643Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.795047705Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:59493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.802670793Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"906c9a8d-1ed0-48a7-b672-3f93619d7b54","remote_addr":"[fd00:1122:3344:101::1]:59493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7556,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:52.806189977Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:52.806200177Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.155957449Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.156086051Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.165176661Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"357f7052-18c5-48bb-8900-b0f19f85016c","remote_addr":"[fd00:1122:3344:101::1]:51947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9033,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.167003749Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:53.167010559Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.422365086Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.422529038Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35265"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.430477501Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"a72c7b94-c40e-490c-b824-36907d0332d1","remote_addr":"[fd00:1122:3344:101::1]:35265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7883,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:53.43368465Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:53.43369145Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.112420883Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.112557645Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43378"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.133069981Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"378fa919-dbbe-4217-b164-bc79fb6e8d99","remote_addr":"[fd00:1122:3344:101::1]:43378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20405,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1729795914.056147, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.137741013Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.137754913Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1298"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.168721131Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.168848803Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50036"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.169233988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.169252209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.188661128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.188690748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.20696002Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.20698941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.224799085Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.224832536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.295033138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.295069538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.353254355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.353294546Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.417685078Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.417723799Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.477440699Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.47747764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.534360887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.534399148Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.53457317Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.534692402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.565665029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.56571431Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.565744681Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.565751831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.565756521Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.565828342Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"68dd35fe-a56e-40b8-adcf-7cc5afb6f127","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\", TimeNotSynchronized), (\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\", TimeNotSynchronized), (\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\", TimeNotSynchronized), (\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\", TimeNotSynchronized), (\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\", TimeNotSynchronized)]","latency_us":396802,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:54.566017405Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"122.511275ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"68dd35fe-a56e-40b8-adcf-7cc5afb6f127\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"68dd35fe-a56e-40b8-adcf-7cc5afb6f127\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.69023813Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.690599755Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.690607425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.709939173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.709981994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.72983966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.729881371Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.747190848Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.747236728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.765371978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.765412118Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.783345565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:54.783384626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.005005117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.005049178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.023794997Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.023830487Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.042237561Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.042286992Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.042540996Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.042731559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.073994731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.074039271Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.074048522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.074057082Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.074081852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.074170083Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a98a6229-a488-4ea4-9760-9a6995d9ff49","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\", TimeNotSynchronized), (\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\", TimeNotSynchronized), (\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\", TimeNotSynchronized), (\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\", TimeNotSynchronized), (\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\", TimeNotSynchronized)]","latency_us":179763,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.074347976Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"166.027413ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a98a6229-a488-4ea4-9760-9a6995d9ff49\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a98a6229-a488-4ea4-9760-9a6995d9ff49\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.241494983Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.241867658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.241879318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.260573677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.260617547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.279075542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.279119313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.297416185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.297457355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.314897194Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.314933435Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.332954362Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.332995983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.351173243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.351221714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.368857456Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.368896967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.386372426Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.386427887Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.38663627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.386752272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418352178Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418403559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418413079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418420099Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418428189Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.418523071Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"249ef40f-0fbe-4884-b9c0-3cea41f9d681","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\", TimeNotSynchronized), (\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\", TimeNotSynchronized), (\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\", TimeNotSynchronized), (\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\", TimeNotSynchronized), (\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\", TimeNotSynchronized)]","latency_us":176856,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.418702863Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"182.866008ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"249ef40f-0fbe-4884-b9c0-3cea41f9d681\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"249ef40f-0fbe-4884-b9c0-3cea41f9d681\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.603065745Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.60333166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.60334082Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.62151283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.621564701Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.638864367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.638917208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.655724757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.655768848Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.672390104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.672424925Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.689988655Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.690029636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.707515806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.707557416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.725016425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.725049996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.742540735Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.742595406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.74283798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.742950692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.773461482Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.773501453Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.773508293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.773516473Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.773523133Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.773646215Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c21b5dd1-68b9-4d27-b590-b2be28090d41","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\", TimeNotSynchronized), (\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\", TimeNotSynchronized), (\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\", TimeNotSynchronized), (\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\", TimeNotSynchronized), (\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\", TimeNotSynchronized)]","latency_us":170450,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:55.77393715Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"216.945869ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c21b5dd1-68b9-4d27-b590-b2be28090d41\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c21b5dd1-68b9-4d27-b590-b2be28090d41\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.991904599Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.992264805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:55.992276665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.010719569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.01076138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.028138608Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.028178529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.045943472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.045999683Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.063763597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.063803728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.081403359Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.08144025Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.098816567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.098854938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.116362038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.116404389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.134235624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.134283364Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.134444477Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.134553298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165305902Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165363123Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165372443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165381273Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165390204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.165495335Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ab906cf6-594f-41c3-b808-b3d32e0ff846","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\", TimeNotSynchronized), (\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\", TimeNotSynchronized), (\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\", TimeNotSynchronized), (\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\", TimeNotSynchronized), (\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\", TimeNotSynchronized)]","latency_us":173428,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.165671068Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"238.53838ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ab906cf6-594f-41c3-b808-b3d32e0ff846\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:51:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ab906cf6-594f-41c3-b808-b3d32e0ff846\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.238982424Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.239243148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.239255688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.257370367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.257402558Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.278055336Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.278091837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.295886231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.295934352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.314724551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.314758642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.332435675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.332473695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.350193828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.350226719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.368419729Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.3684507Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.386258504Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.386300295Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.386503298Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.38663235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.469141792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.469200473Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.469208533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.469216283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.469220613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.559768729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.564028145Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.568317571Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.572350633Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.576434896Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.581068647Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.585603787Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.589944354Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.594533285Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:56.599107105Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:56.604670251Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"318.455892ms"} {"msg":"Profile for oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.674666758Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.674769969Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.680643537Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.680775614Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.700134467Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.700327849Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.705115772Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.705341363Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.705382483Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:51:59.709643519Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 42.719791ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.822098307Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 42.033471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.829684995Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 44.748379ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.842289841Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 50.926365ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.848797815Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 52.940772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.855163898Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 141.274062ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.872104277Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 68.222027ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:51:59.880692487Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 127.867492ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.017246915Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 140.332515ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.184718261Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 119.142898ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.321610213Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 56.697475ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.567934186Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 223.939624ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.650179357Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 564.458395ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.727261746Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 251.345428ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.847483501Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 158.56232ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:00.878824983Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 599.570562ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:01.006961949Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 200.018995ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:01.416123448Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 847.234124ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:01.695758698Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 481.08735ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.500676474Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 400.410731ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.689777627Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 778.529725ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.785329642Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 476.304927ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.798190652Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 902.052274ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.809325351Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 734.767209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:02.995227528Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 442.13145ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.102196337Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 663.35944ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.289052114Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 705.26552ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.558285476Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 951.705897ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.576181352Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 1.064386426s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.724085648Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 979.233551ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.740995298Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 742.869692ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:03.963296616Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 813.15798ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:04.275150218Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 664.344439ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:04.539804633Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 965.570033ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:04.717109052Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 684.847409ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:04.732010164Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 764.60305ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:04.800382114Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 689.583006ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.101152625Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 513.841869ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.220432658Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 818.055163ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.430825072Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 520.253474ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.579092007Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.12986286s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.697717387Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 966.655354ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.747348722Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 890.614376ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:05.807734786Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:05.900150036Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:05.900436972Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"d1760acb-96f9-4be9-8333-2b92177ca2b4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58305","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 573.12392ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.110401466Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 1.307427958s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.260772742Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 799.372509ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.694640832Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 1.388717937s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.70895653Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 523.667759ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.723867225Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.17644681s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:06.839486216Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.305809197s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:07.257226504Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 1.096990711s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:07.505918933Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 678.84457ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:07.581895387Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 904.848896ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.029175541Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 669.420047ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.110223157Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac\") failed: Property not found. retry in 1.130346707s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.272098116Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 595.524353ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.576288779Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb\") failed: Property not found. retry in 1.185863186s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.615429041Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270\") failed: Property not found. retry in 814.959999ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.791371561Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 866.72982ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:08.948152196Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.246709166s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:09.185567836Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.004039916s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:09.831254303Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.093849498s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:10.446259812Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 856.280105ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:10.847489358Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:12.059921135Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"333.060224ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.154292289s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:12.08785795Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.002222675s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:12.136699051Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 621.490074ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:13.152473667Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.013937447s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:13.250576203Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.24921182s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:13.785192026Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.40609588s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:14.276327409Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.034048378s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:15.050679745Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.15020206s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:15.696131155Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 858.543431ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:16.097245839Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 1.142544751s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:16.859437229Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.231822602s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:16.967144636Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948\") failed: Property not found. retry in 987.133366ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:18.014161775Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee\") failed: Property not found. retry in 1.10357148s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:18.211812479Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.34067028Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_7ea30bac-13f2-4e1c-b902-1f3b630fdfdb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.340716819Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_df540489-9518-4c44-948f-77fa82fe99ac"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.340724797Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_096da1d3-d66d-49ff-8236-d77eb3631270"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.340729747Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_ce74b7b7-6a3e-46ac-8376-85d9df951948"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.340736047Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_cc1d03a8-51c6-471a-9d2b-b6a143151eee"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.340775207Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.341853767Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.344223824Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b96802f1-3c91-47ff-9269-276c3a18f4a3","remote_addr":"[fd00:1122:3344:101::1]:50036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":23044662,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.377699579Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43294"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:19.405127209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3720"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:20.355917697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3738"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.880079432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.880196977Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"dbd81e38-72fe-4866-94c1-378d570b4df2","remote_addr":"[fd00:1122:3344:101::1]:43294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":12468436,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:31.884882731Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.325260065s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.917289561Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.917447532Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40024"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.917859053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.917865128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.940277707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.940336471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.961524148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.961568821Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.984205723Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:31.984252477Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.008171224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.008219318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.033876729Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.033994297Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.05661416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.05665931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.078277213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.078315342Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.098854275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.098898161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.180639542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.180687814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.263747707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.263801117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.352406852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.352458247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.431945101Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.432060808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.51399356Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.514046089Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.597229481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.597291038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.679639386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.679765208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.74910998Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.749162949Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.812426098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.812476395Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.879583534Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.879634778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.942560306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:32.942608241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.00922743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.009285855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.009538704Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.009745113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114057803Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114124722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114131341Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.11413856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114144344Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114149838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114155491Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114160863Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114166047Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114170969Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114176248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114181263Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114196526Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114202104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114207026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114212033Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114216983Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114222289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.114228748Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.114330122Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"47aad5d0-fcb6-498b-8cff-59bfae441825","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized)]","latency_us":1193465,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.11449294Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"89.53126ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"47aad5d0-fcb6-498b-8cff-59bfae441825\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"47aad5d0-fcb6-498b-8cff-59bfae441825\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.2049909Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.205355661Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.20536242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.22823389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.228282284Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.250037882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.250084739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.272670953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.272709851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.291912663Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.291947989Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.311467052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.311502087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.330280058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.330313274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.349074847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.349112628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.367225505Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.367260503Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.385859377Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.385894103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.403817327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.403854275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.422152041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.422193245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.440028323Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.440057367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.458282935Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.458324654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.478771636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.478799996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.498369779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.498399376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.51732979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.517363897Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.535648393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.535677335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.556130295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.556158167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.574938916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.574969479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.594061775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.594101236Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.594359032Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.594481828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690078417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690124074Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690131902Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690137518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690144024Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690149452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.69015579Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690161181Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690167678Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690172731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690177981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690184619Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690191125Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.69019676Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690208947Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690214253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690219419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690224511Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690229536Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.690361294Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"89275a5f-a8be-4af9-b7f8-21d90ec7d0e2","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized)]","latency_us":483932,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:33.690584816Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"90.811685ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"89275a5f-a8be-4af9-b7f8-21d90ec7d0e2\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"89275a5f-a8be-4af9-b7f8-21d90ec7d0e2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.783006289Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.783277109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.783287816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.802167461Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.80220198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.820297658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.820331567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.840460375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.840504664Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.85992387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.859960049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.877810071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.877843953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.897503539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.897542379Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.915667705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.915701906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.936015223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.936051458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.954562533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.954597071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.976061324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.976101665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.993521507Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:33.993552286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.011731704Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.011790055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.03388675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.033921729Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.05271344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.052747894Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.07102819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.0710624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.08713105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.08716407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.106909132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.106943669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.125371199Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.125401218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.14501591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.145050279Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.164078394Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.164114029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.164317039Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.164453034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254535625Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254581928Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254608882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254615304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254622729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.25463096Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254637185Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254644217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.25465136Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254658214Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254664205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254670252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254677855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.25468513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254691374Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254698424Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254705671Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254717136Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.254722734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.254815866Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6fc97808-e74f-41c6-bed7-4fb0995337ab","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized)]","latency_us":470466,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.255030755Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"176.225026ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6fc97808-e74f-41c6-bed7-4fb0995337ab\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6fc97808-e74f-41c6-bed7-4fb0995337ab\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.432901847Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.43316571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.433171973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.453348804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.453387036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.472930806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.472966619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.491753934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.491798091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.510841818Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.510876319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.529732762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.529763353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.548199908Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.548235871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.566286309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.56632126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.58474819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.58478766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.603723088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.603756914Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.623522314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.623561268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.642340777Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.642382234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.660265152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.660297187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.678686484Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.678736134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.697241842Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.697274805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.715639557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.715673936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.734614151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.73464596Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.753043332Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.753073557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.771664578Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.771696932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.791553607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.791587705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.809345276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.809384877Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.809637076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.809754322Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902417214Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902469724Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902475818Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902480712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902486084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902511977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902518325Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902523396Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.90252801Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902532782Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902537553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902543957Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902548888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902555Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902559941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.90256476Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902571153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902574594Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902579328Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:34.902653037Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"435d8376-fed1-4545-9933-3edd4da6a205","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized)]","latency_us":468407,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:34.902876446Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"179.357769ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"435d8376-fed1-4545-9933-3edd4da6a205\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"435d8376-fed1-4545-9933-3edd4da6a205\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.08377435Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.084040135Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.084047167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.108423376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.108459883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.127748944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.127779339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.14655518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.146595981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.169151267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.169192996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.193436993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.19355433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.214909342Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.214954231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.238389298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.238425973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.261785691Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.261827335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.284993792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.285045786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.309001253Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.309039343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.332145281Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.332188181Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.354116408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.354152447Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.375398492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.375444271Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.399988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.40009767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.419623944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.41967305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.441860951Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.441901601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.461963778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.462005779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.484514209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.484556192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.506261299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.506301472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.527501892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.527551214Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.527734564Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.527867204Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633232365Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633290603Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633298253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633302209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633307281Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633313731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633319319Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633324419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633348879Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633354325Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633359154Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633363982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633370301Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633375335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633380314Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633386829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633391901Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633398267Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633401867Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.6334848Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c6be91c8-1af4-41fb-b738-c6248b8a0bc8","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized)]","latency_us":548132,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:35.633633527Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"173.266021ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c6be91c8-1af4-41fb-b738-c6248b8a0bc8\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c6be91c8-1af4-41fb-b738-c6248b8a0bc8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.808612868Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.808912169Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.808921132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.8349109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.834966616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.856346627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.85639205Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.877850672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.877893207Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.897537309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.897573648Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.918992703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.919041312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.938356775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.938397388Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.957085458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.957127074Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.976112226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.976150636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.9973795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:35.997419242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.021579685Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.021839308Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"baf6b513-e969-492c-9d2e-0b3141971636","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63185","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.021967156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.021975239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.046282641Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.046326048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.065718243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.065761546Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.088230343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.088279066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.109825177Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.109902625Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.134928521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.134971844Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.156817762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.156859012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.179608146Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.17964994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.201432722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.201481529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.223472596Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.223518319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.245504283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.245560703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.245724768Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.245868152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343678659Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343737113Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.34374346Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343748907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.3437556Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.34376206Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343767441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343790861Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343797592Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343802504Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.34380752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.34381247Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343817345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343823383Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343828361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343834379Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.34383934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343844223Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.343847459Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.343940103Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"408abcdd-c448-4ae2-996c-9a96ad6ef118","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized)]","latency_us":533724,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:36.344103756Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"214.013064ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"408abcdd-c448-4ae2-996c-9a96ad6ef118\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"408abcdd-c448-4ae2-996c-9a96ad6ef118\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.558993513Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.559345912Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.559354339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.582598241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.582636707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.604630128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.604669898Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.625278944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.625317869Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.647230776Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.647268774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.670013402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.670080434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.692358237Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.692400454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.715529152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.715569241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.737528227Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.737564612Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.760268349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.760308108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.783459291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.783505436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.804382389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.804417977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.828218763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.828257679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.850819361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.850858718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.872451081Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.872486248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.894766853Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.894800979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.919047128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.919081572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.941996063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.942029428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.965375883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.965408565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.985227542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:36.985266299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.007777393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.007825797Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.00802711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.008211153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119500669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119572783Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119583499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119592986Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119601658Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119610189Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.11961888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119627964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119638719Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.11964784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119678984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119687103Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119693048Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119767668Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119774868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119783212Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119789596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.119794968Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.11980019Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.119844525Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"aae59b0a-2f34-4291-b264-f8a1f7a2ad58","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized)]","latency_us":559168,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.120008647Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"314.78431ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aae59b0a-2f34-4291-b264-f8a1f7a2ad58\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aae59b0a-2f34-4291-b264-f8a1f7a2ad58\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.436367124Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.436753838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.436760747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.460794559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.460842007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.480738295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.480778824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.501690341Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.501805813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.523348621Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.523386993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.54482626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.54493469Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.565712277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.56576114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.587270768Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.587309075Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.60954796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.609589689Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.631017515Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.631064269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.652404225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.652445007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.674560785Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.674601858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.698395038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.698433467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.718797977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.718840475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.739556939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.739594665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.760976551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.761024607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.782976236Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.783028531Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.803215099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.803263756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.825124898Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.825167029Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.84506722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.845108996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.866621186Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.866683727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.866956831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.867104105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.96669392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966738049Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966743946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966747865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966751802Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966755703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966758824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966762724Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966766409Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966770159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966773825Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.9667775Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966781391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966802728Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966806816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966810753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966815356Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966819079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.966821967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:37.966899236Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"54cf4195-6b4b-41ec-afd7-99d04e5631ae","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized)]","latency_us":528965,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:37.967109909Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"399.962847ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"54cf4195-6b4b-41ec-afd7-99d04e5631ae\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"54cf4195-6b4b-41ec-afd7-99d04e5631ae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.369703229Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.370013743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.370020221Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.392220747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.392252201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.414424672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.414462884Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.434843488Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.434877051Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.454215741Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.454244261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.475192018Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.475222788Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.495263107Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.495294674Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.514377663Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.51441943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.534215703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.534245346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.553729874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.553764253Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.575836656Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.575866028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.595672565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.595767339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.61617532Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.616204149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.639079828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.639111713Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.659784573Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.659814976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.681650494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.681680757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.704643709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.704682437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.724717278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.724743782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.745334288Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.745363576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.765963745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.765992536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.790060748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.790096786Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.790373264Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.790494185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893335062Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893383166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893390937Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893395091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893401372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893406678Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893412106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893418791Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893424153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893429159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893433116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893438254Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893443598Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.89344866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893454032Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893459104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893481473Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893487117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893493342Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:38.893688029Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7c3cdf38-d35e-44e7-b00c-f6839296a643","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized)]","latency_us":522396,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:38.893849769Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"513.756593ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c3cdf38-d35e-44e7-b00c-f6839296a643\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c3cdf38-d35e-44e7-b00c-f6839296a643\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.41063672Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.411006433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.411015087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.435221124Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.435262252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.454608035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.454645827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.478737099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.478798403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.500101187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.500138518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.521338489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.521376534Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.543155587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.543192205Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.56463803Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.564675634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.584759796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.584796097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.603839417Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.603876234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.625041456Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.625075732Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.645512143Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.645615728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.666781307Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.666817091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.687009518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.687054265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.709377013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.709416905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.728228408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.72826784Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.749378988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.749414931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.769178429Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.769235861Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.791620918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.791659366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.812680349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.812721656Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.835013627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.835057286Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.835344935Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.835481277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935492642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935530967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935537886Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935550449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935554395Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935558324Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935563331Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935567437Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935572134Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935575949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935580506Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935584284Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935588999Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935592065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935594953Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935597849Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935600699Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935604346Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935618288Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:39.935663616Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"97dc21c2-6a57-4ef7-857d-2aa22414fd2c","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized)]","latency_us":523455,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:39.935825966Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"395.618051ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97dc21c2-6a57-4ef7-857d-2aa22414fd2c\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97dc21c2-6a57-4ef7-857d-2aa22414fd2c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.333214097Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.333559034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.333568906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.356356035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.356390658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.378442677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.37847234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.39913467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.399165918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.421358932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.421398205Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.44327411Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.443303857Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.464952356Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.464980668Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.485486199Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.485515945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.506159652Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.506186381Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.526152049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.526179012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.546518764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.54654687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.565486289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.56551387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.58659641Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.586634154Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.609061964Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.609097046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.629583191Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.629614241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.650912829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.650940018Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.671422002Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.671450248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.692088411Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.692120325Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.713987076Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.714026714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.733876601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.733904783Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.755862171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.755892284Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.756101256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.756206511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866144458Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866188868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866194737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866198412Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866203513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866210065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866225441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866230653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866235509Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866240553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866247106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866252028Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866257316Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866260934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866265763Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866269166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.86627391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866278485Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866283201Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:40.866346174Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"33c2dabc-14e5-46e6-a65e-1d937b291bfd","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized)]","latency_us":531541,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:40.866534473Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"343.606627ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33c2dabc-14e5-46e6-a65e-1d937b291bfd\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"33c2dabc-14e5-46e6-a65e-1d937b291bfd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.211867835Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.212167165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.212193565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.234709595Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.234749777Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.254009289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.254041212Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.272959048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.273068818Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.291551009Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.291585463Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.309985045Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.310024468Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.328278401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.328311644Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.34653161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.34656942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.367172272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.367208309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.386836143Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.386871965Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.405713079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.405768523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.425294759Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.42533226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.444156569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.444201288Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.463792576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.463832899Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.483153309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.483201188Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.501735514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.501769133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.520058533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.520092199Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.538791373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.53882502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.5586326Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.558667523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.57722479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.577257153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.595831164Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.595944988Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.596167516Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.59628719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.69876381Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698800645Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698805698Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698810545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698814529Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698818232Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698822151Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698826061Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698830729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698841979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698845129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.69884799Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698850961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698854721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698858498Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698861471Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698865071Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698868821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.698872561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:41.698914505Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"012ee179-7df7-420b-a46d-bc573e68551e","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized)]","latency_us":485661,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:41.69906883Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.065031966s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"012ee179-7df7-420b-a46d-bc573e68551e\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"012ee179-7df7-420b-a46d-bc573e68551e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.768348124Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.768648232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.768655011Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.792192396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.792239516Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.814636774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.81468289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.838425919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.838464094Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.858870799Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.858906368Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.878600269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.878635173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.901032113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.901071366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.921562166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.921600013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.941578063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.941610736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.961734179Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.961771286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.98132887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:42.981371706Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.00131312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.001347611Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.024428063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.024471685Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.043530606Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.04356927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.061493817Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.061532265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.084548895Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.084592143Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.104675641Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.104709748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.125921161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.125957031Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.145552814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.145587315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.165752742Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.165786924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.184058441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.1841015Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.184352605Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.184461403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287745205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287780794Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287786634Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287790844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287794669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.28779869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.28780244Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287806134Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287810015Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287813653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287816578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287827641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287831654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287835488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287839023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287842585Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.28784641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287849344Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.287852194Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:43.287897955Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4bf50624-8cea-4e1a-b093-1d1eb33c82e6","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized)]","latency_us":518011,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:43.288064372Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.059928779s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bf50624-8cea-4e1a-b093-1d1eb33c82e6\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4bf50624-8cea-4e1a-b093-1d1eb33c82e6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.352806228Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.353088504Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.35309622Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.375994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.376027075Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.398564329Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.398608017Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.418649778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.418687588Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.43914046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.439168624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.458170242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.458202885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.479169344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.479198707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.497867625Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.497894992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.517298157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.517326976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.537671284Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.537700975Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.558049249Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.558083093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.578551693Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.57859509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.599713951Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.599746951Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.622504344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.622535919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.64472239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.644754293Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.666820634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.666850822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.687869563Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.687907195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.708257576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.708288692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.727828444Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.72785897Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.7493299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.749361813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.771817724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.771847012Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.772048531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.772153655Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870822738Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870869305Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870876767Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870881717Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870885261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870890192Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870895133Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870900177Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870904967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870910265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870915524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870919209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870922734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870927946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870938896Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870944193Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870949462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.87095429Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.870959221Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:44.871092264Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"64d0fb52-44d6-4e1c-a4e3-3862bfb170f5","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized)]","latency_us":516747,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:44.87129218Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"626.977329ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"64d0fb52-44d6-4e1c-a4e3-3862bfb170f5\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"64d0fb52-44d6-4e1c-a4e3-3862bfb170f5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.501682197Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.502024559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.502035987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.524154522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.524198745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.546267218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.546303782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.566858668Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.566899806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.586646132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.586745651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.60666709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.606702556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.626079684Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.626113987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.64511647Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.645151093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.663990214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.664024106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.684837344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.68487219Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.706656756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.706694097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.727496839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.727533467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.751333727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.751383472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.773446275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.773488294Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.793061632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.793098843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.813536238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.813572903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.832893039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.832935921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.853503162Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.85354054Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.877720036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.877769218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.901248027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.90128731Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.924167899Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.924211577Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.924397748Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:45.9245221Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029051668Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029104225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029131367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029137011Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.02914092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029146086Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.02915127Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029156098Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029160739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029165493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.02917032Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029173639Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029177118Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029180212Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029183662Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029187083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.02919044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029197021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.02920213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:46.029206968Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e236c280-a6e4-44a4-8722-150b75f1bb0f","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized)]","latency_us":525961,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:46.029361827Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.44411084s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e236c280-a6e4-44a4-8722-150b75f1bb0f\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e236c280-a6e4-44a4-8722-150b75f1bb0f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.479386759Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.479659942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.479666345Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.502580164Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.502625775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.522411805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.522450703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.542970681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.543086309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.561467048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.561501979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.580154841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.580188179Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.598809882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.598847786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.61682603Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.616862884Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.6366239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.636665986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.654996842Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.655027976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.673825349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.673865016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.693477091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.693514771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.712100564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.712149953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.73184357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.731894627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.751280318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.751325225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.769696743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.769761001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.788641831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.788759703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.809600529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.809651952Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.828511474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.828561743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.846597936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.846636983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.865364943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.865421138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.865611341Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.865740362Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961361159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961417841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961422285Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961426372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961437557Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961456551Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961460076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961463095Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961466076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961469329Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961472348Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961478536Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961481751Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961484892Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961489054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961492729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961499405Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961502846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961509418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:47.961512643Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"722ee930-5acd-4556-b748-de46683f7c67","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized)]","latency_us":480675,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:47.961648235Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.114481967s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"722ee930-5acd-4556-b748-de46683f7c67\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"722ee930-5acd-4556-b748-de46683f7c67\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:48.31490151Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.693450535s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.082578076Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.08286415Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.082871087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.106914028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.107012355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.126263996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.126291859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.145632273Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.145665855Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.163662579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.163692092Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.181624735Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.181647236Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.199387617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.19942824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.217233361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.217337422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.234898027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.23492455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.253212514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.253246274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.271287632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.271320371Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.28975319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.289787766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.309280947Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.309316358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.327717113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.327763539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.347379697Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.347414272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.365863149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.365896909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.384473838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.384503453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.403166455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.403196146Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.421421033Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.421456986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.440338597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.440374598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.458540357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.458583248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.458785592Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.458990644Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552659843Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552731412Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552742006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552754279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552760982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.55279521Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552806948Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552817523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552836208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552855727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552884884Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552895796Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552904824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.55291359Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552924315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552930907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.55294Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552949488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.552956397Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:50.552968313Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2d3461e7-e72d-495c-8c79-2bd3792c1d3e","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized)]","latency_us":469020,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:50.553266344Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.584416775s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d3461e7-e72d-495c-8c79-2bd3792c1d3e\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d3461e7-e72d-495c-8c79-2bd3792c1d3e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.142701141Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.143157373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.14316958Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.167317992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.167353129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.186699417Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.186733055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.206549582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.206585124Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.225122769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.225153407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.243060792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.243090651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.260964098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.260990892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.279559692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.279588557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.297605604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.297638914Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.316175287Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.316211681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.335753351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.33578819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.354286121Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.354318361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.374168983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.374213027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.391953498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.391985271Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.41033571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.41037608Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.42833827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.428378423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.446864941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.446903332Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.465619254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.465669186Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.484836755Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.484870983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.504238214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.504275799Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.522744393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.522781256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.522967437Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.523087102Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617453974Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617514594Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617528188Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617547031Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617561029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617567667Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617573864Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617580679Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617607098Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617615282Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617629682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617635758Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.61764774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.61765524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617663499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617669499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617677158Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617694691Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.617705706Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:52.617851412Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"aa35177a-11a8-45cc-94ad-35f996984fbc","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized)]","latency_us":473640,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:52.618018675Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.38158034s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa35177a-11a8-45cc-94ad-35f996984fbc\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa35177a-11a8-45cc-94ad-35f996984fbc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.007313499Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.007647975Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.007659264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.030139077Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.030180814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.051931607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.051964889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.072006166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.072036607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.094144507Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.094174283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.114421688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.11444867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.134184554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.134223235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.152520221Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.152551778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.170909244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.170941476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.1891998Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.189233955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.207570582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.207606442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.225660237Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.225689123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.244032707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.244065305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.261900763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.261929798Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.280432215Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.280464598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.298243792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.298281959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.316267157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.31629832Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.334657785Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.334690842Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.352756338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.352794739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.370820584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.370850687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.389663956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.389703547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.389872047Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.38998167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481845798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481907984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481917856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481930325Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481938472Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481945138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481951456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481959688Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481966035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481972475Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.481981747Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482006432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482026204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482032673Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482038429Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482053468Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482061211Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482067821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482072461Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:55.482112305Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"91695b72-ea33-4800-8dfa-4760a451f5c2","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized)]","latency_us":473403,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:55.482268786Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.683850406s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91695b72-ea33-4800-8dfa-4760a451f5c2\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91695b72-ea33-4800-8dfa-4760a451f5c2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.1775165Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.177862211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.177877924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.201134626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.201176579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.219516598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.219560117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.237993006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.238027047Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.255603114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.255639443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.275260184Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.275292988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.293634468Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.293669315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.312070813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.312106766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.329686633Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.329723824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.348330559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.348376357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.366649988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.366681246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.384378324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.384416968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.402684796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.402721453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.420613852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.42064267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.438853744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.438881907Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.457705425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.457739034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.475562487Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.475602557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.493379391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.493405716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.511951386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.51198447Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.529899737Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.529930779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.54813275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.548160932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.548342809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.548452752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.639913187Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.639976432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.639984139Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.63999132Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.639996383Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640003751Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640015339Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.64002177Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640029495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640035449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640041468Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.64005419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640061737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640067643Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640089927Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640096669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640107806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.640114331Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.64012021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:52:59.640175757Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b09a2375-6e38-4fb1-a5b9-a0c40182cad6","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized)]","latency_us":461246,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:52:59.64034961Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.661339567s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b09a2375-6e38-4fb1-a5b9-a0c40182cad6\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:52:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b09a2375-6e38-4fb1-a5b9-a0c40182cad6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.315889907Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.316469066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.316477401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.337855366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.337903733Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.356885594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.356934739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.37746061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.377511893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.395150692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.395208611Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.414390795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.414436264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.432517637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.432569387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.450434255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.452723972Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.469570673Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.469611512Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.487480623Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.487541542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.506313469Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.50635892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.525440094Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.52548608Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.544481485Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.544538449Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.563089808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.563152593Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.581691233Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.58174002Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.599632781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.599685807Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.618206645Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.618248795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.636673705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.636722972Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.655810794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.655855654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.674352687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.674397838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.693281442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.693349956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.693634978Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.693784091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789346229Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789407805Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789413468Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789426021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789431252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789435425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789440309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789454287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789459294Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789465275Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789470225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.78947599Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.78948199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789488347Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789498051Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789503385Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789507407Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789524366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.78952956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:04.789571345Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"522f1ab1-25a2-4d98-8971-3622482f19b8","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized)]","latency_us":472218,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:04.789715319Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.499150085s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"522f1ab1-25a2-4d98-8971-3622482f19b8\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"522f1ab1-25a2-4d98-8971-3622482f19b8\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:05.089165464Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"3.770091019s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:06.137849575Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:06.137965404Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"577cc74e-7b8f-4496-a2c3-27951fc99c57","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51722","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.302420503Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.302676688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.302684629Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.324665697Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.324712376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.34390986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.343946705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.361884548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.361927682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.380965104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.380997551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.399537311Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.399574511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.417828404Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.417864639Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.437574523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.43760817Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.455890214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.455925126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.475246468Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.475280209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.493800482Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.493834711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.51269277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.512728874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.530963021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.53099634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.549008387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.549049065Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.566752435Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.566856731Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.585333884Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.58536635Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.603675268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.603710893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.622282483Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.622318764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.640830414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.640866836Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.659485265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.659515359Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.677517263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.677549129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.677767279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.677898044Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774517129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774565682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774571964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774587968Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774592121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774603774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774609315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774623265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774632508Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.77463789Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774643797Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774649366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774653969Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.77466025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774681287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.77468845Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774694497Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774699963Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.774711692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:09.774848343Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1a7c6e5a-e27c-4bc3-a97e-6d8e3525fe67","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized)]","latency_us":471049,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:09.775047649Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"5.774892346s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a7c6e5a-e27c-4bc3-a97e-6d8e3525fe67\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a7c6e5a-e27c-4bc3-a97e-6d8e3525fe67\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.567232104Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.567768746Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.56777694Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.592290276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.59234108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.612168337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.612273778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.630708308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.630766556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.648556521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.648612182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.668079165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.668127579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.685968859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.686020966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.704091048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.704145996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.722276592Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.722316624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.740890773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.74093177Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.758867093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.758914223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.776805624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.776850635Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.795299761Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.795343665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.81317996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.813225476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.831444801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.831494386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.849941907Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.849982773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.868277502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.868318556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.886199938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.886247845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.904505872Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.904545894Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.92222999Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.922292185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.941432956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.941487482Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.941688017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:15.941833828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039475207Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039525907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039549842Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.03955663Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039561589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039572174Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039577302Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039581427Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039585345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039590389Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039596211Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039601255Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039607443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039616518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039626409Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039632681Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039637321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039658829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039663957Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:16.039719411Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"912d9553-7cea-4397-b24c-836783bd43c8","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized)]","latency_us":470994,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:16.039869328Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"5.384011203s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"912d9553-7cea-4397-b24c-836783bd43c8\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"912d9553-7cea-4397-b24c-836783bd43c8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.440569966Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.44084327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.440850349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.465890307Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.46593138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.489220098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.48926103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.509291274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.509333997Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.52898267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.529021418Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.547675494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.547713192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.569643562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.569691442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.589132786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.589169123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.608337626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.608395114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.630698428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.630743757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.652069324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.652107528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.671570021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.671605928Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.691113188Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.691148786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.711401496Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.711444386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.732021847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.732055082Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.75132554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.751357809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.772210195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.772245398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.791839819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.791879195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.81177447Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.811812964Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.830732006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.830766759Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.850516669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.850560329Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.850729672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.850851314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.945992083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946051859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.94605964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946075728Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946089707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946095697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946100976Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946112132Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946124441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946129879Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946135335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946140969Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946146145Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946159252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946165045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946176052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946180411Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946185258Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946194548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:21.946267778Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f39e38e0-4634-4649-a1fa-53ee6b6318a5","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized)]","latency_us":504234,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:21.946484395Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.199389235s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f39e38e0-4634-4649-a1fa-53ee6b6318a5\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f39e38e0-4634-4649-a1fa-53ee6b6318a5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:23.945512732Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"10.742205769s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.157652524Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.157992602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.158001114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.182594869Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.182655554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.205434032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.205482127Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.227459903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.227511521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.248264234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.248313331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.270879093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.2709345Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.291097758Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.291153174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.312944003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.312999513Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.332744669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.332798904Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.352707453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.352757732Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.374049567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.374097727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.39591846Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.395970267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.416926363Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.416975967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.437098283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.437153728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.456270634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.456320285Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.475563442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.475617865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.494134735Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.494181564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.519026747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.519077035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.541949659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.541997744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.566377388Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.566427891Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.588556378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.588637351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.588815273Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.588961778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692132217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692199175Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692206065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692217466Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692223138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692238775Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692311159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692317272Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692322906Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692329985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692336126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692342707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692349541Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692356591Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692362207Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692379673Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692386161Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692393501Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.69239878Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:26.692498259Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"61ec7458-c153-4249-98d2-816d267eee09","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized)]","latency_us":533223,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:26.692657402Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"9.669977543s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61ec7458-c153-4249-98d2-816d267eee09\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61ec7458-c153-4249-98d2-816d267eee09\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.253931192Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53118"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.254020032Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ce994045-8dc0-4234-aa80-720a3ed84d93","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53118","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.390195762Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.390481161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.390489176Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.413114903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.415849239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.436075103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.436119213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.455537613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.455596658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.47410927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.47416614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.49383249Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.493943547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.512093939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.512147349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.53180963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.531854077Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.551154509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.551210882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.570708883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.570766062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.589920064Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.589966143Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.609088383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.609139712Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.62784448Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.627891975Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.647416124Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.647465306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.667291569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.667405542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.68661216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.686672217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.707035306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.707079847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.725789218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.725832888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.744194973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.744252593Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.76275784Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.762805954Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.780962062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.781009735Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.781236942Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.781366862Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874421454Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874502727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874511905Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.87452923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874536384Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874560122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874571766Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874578853Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874584319Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874590956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874600866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874606491Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874614863Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874620892Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.87462616Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874634635Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874639782Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874646701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874655026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:36.874725433Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3ae4e879-bafc-4143-a1f3-5caaee69d5ac","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized)]","latency_us":483118,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:36.874956366Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"8.968979161s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ae4e879-bafc-4143-a1f3-5caaee69d5ac\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ae4e879-bafc-4143-a1f3-5caaee69d5ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.869546051Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.869860821Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.869878175Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.895353017Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.895472194Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.916405394Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.916517762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.93602774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.936080616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.95481333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.954857328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.975959017Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.976016364Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.993888319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:45.993937961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.013144452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.013187794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.032407663Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.032466408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.053314498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.053371405Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.072410924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.072455991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.092288353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.092333692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.111927825Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.112042573Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.130474423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.13052112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.149707115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.149759372Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.167882002Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.167939687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.18670255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.186743932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.205504586Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.205553272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.224638105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.224681072Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.243893775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.243933085Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.262119832Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.262183311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.262503785Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.262654313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.35719434Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357248705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357265984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357274731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357283178Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357303841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.3573124Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357321054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.35733446Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357342963Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357357457Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357365632Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357372101Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357392099Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357398427Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357406283Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357414149Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357420308Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357428596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:53:46.357507282Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"dad48b12-2b1e-4999-a7f0-6e29b374b7a1","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized)]","latency_us":486562,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:46.357747582Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"13.658073263s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dad48b12-2b1e-4999-a7f0-6e29b374b7a1\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:53:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dad48b12-2b1e-4999-a7f0-6e29b374b7a1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:53:49.795243996Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"8.86987886s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.053876875Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.054192077Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.054206496Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.076839095Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.079246678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.093746275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.093796086Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.114250455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.114301548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.131548413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.131595786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.149708198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.149748352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.167775079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.167830111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.184730433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.184785147Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.202539068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.202579466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.219805549Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.219859934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.238165434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.238212404Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.255679549Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.255726538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.27380376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.273850148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.289895347Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.289944482Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.308099968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.308152637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.325690044Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.325734257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.341654914Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.341698106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.360437149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.36048034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.377321399Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.377372831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.395529471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.395573825Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.414013695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.414071727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.414357782Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.414561391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512674586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512729712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512738655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512748358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.51275449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512761081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512782653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512790406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512804994Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.51281692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512822873Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512828723Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.51283432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512840432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512846404Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512857138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512861713Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512872598Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.512878908Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:00.512909358Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a9664403-223d-4802-92be-bc22754d149f","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized)]","latency_us":457708,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:00.513154006Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"7.0389774s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9664403-223d-4802-92be-bc22754d149f\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:54:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9664403-223d-4802-92be-bc22754d149f\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:06.367206637Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44238"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:06.367274485Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"a385fb39-d59e-4c4f-bc4e-c6a12edec4b5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44238","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.572396417Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.572691321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.572714215Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.596318771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.596375876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.615590993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.615629469Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.632524843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.632580053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.650498064Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.652371695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.668296151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.668342792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.685748145Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.685795704Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.703929977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.703973946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.721432632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.721473423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.739483656Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.739536851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.758478047Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.75852106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.77658069Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.776626263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.794294401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.794340217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.811745174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.811793916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.830490304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.830546198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.848188602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.848231736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.868627806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.868671298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.886820747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.886864023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.904548944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.904597067Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.920471061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.920518255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.938441387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.938505016Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.938654578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:07.938773174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034093108Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034140003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034154909Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034166056Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034171129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034180373Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.03418641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034191557Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034209379Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034215885Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034220666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034227079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034231082Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034240119Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034245089Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034250292Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034255148Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.03426008Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034264017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:08.034300581Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9e719397-9c91-498a-9ad3-4b41f479b2fc","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\", TimeNotSynchronized), (\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\", TimeNotSynchronized), (\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\", TimeNotSynchronized), (\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\", TimeNotSynchronized), (\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\", TimeNotSynchronized), (\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\", TimeNotSynchronized), (\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\", TimeNotSynchronized), (\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\", TimeNotSynchronized), (\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\", TimeNotSynchronized), (\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\", TimeNotSynchronized), (\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\", TimeNotSynchronized), (\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\", TimeNotSynchronized), (\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\", TimeNotSynchronized), (\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\", TimeNotSynchronized), (\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\", TimeNotSynchronized), (\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\", TimeNotSynchronized), (\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\", TimeNotSynchronized)]","latency_us":460589,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:08.034532471Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"8.551048715s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e719397-9c91-498a-9ad3-4b41f479b2fc\", \"content-length\": \"124\", \"date\": \"Thu, 24 Oct 2024 18:54:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e719397-9c91-498a-9ad3-4b41f479b2fc\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:13.766166038Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"17.220706897s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 26818eb7-96aa-486e-ac98-9a5f4e70076d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d4b09213-5961-4095-a130-f1ddc420cb29 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 06542437-1810-4294-a723-9ed308666b19 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 638a0656-b644-4d47-9872-6e9a903f4544 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8ab2b755-55cf-4e78-9abc-38b1d56687dc, kind: Service { id: 638a0656-b644-4d47-9872-6e9a903f4544 }, name: Name(\"ntp-638a0656-b644-4d47-9872-6e9a903f4544\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: df540489-9518-4c44-948f-77fa82fe99ac (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: ce74b7b7-6a3e-46ac-8376-85d9df951948 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: cc1d03a8-51c6-471a-9d2b-b6a143151eee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: 096da1d3-d66d-49ff-8236-d77eb3631270 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: e40b6155-d379-4874-9119-51a498c972c1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b130f689-a354-4ce5-be2b-20413d917375 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b33ec370-f410-4564-a49d-41d565d13ca6 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d3774998-58a6-4b89-8d53-ad0f914d458d (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2394e76f-ffd7-4010-81b9-fd582588a6a2 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 80ad703a-fbb4-4254-a4ab-a4358356c4a5 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 80a8c73c-5c3b-49f3-8129-27ed1a596c1d (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13149332-2b9e-43f3-86ad-209fb5b70e55 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External } } } }, OmicronZoneConfig { id: 89558561-0976-4295-a611-2d4619e6b7a1 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External } } } }, OmicronZoneConfig { id: b27d58f3-b052-4217-923a-62b3896e867a (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc09e276-dc62-4ef0-a414-a788a80131ad (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 153af4c7-3f5c-4702-aad3-4d1551a443cf (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.609793853Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.610095048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.610107676Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 26818eb7-96aa-486e-ac98-9a5f4e70076d, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.634272101Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.634314767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4b09213-5961-4095-a130-f1ddc420cb29, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.655971376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.656008155Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06542437-1810-4294-a723-9ed308666b19, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.676409698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.676443157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: df540489-9518-4c44-948f-77fa82fe99ac, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.696938818Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.697052968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce74b7b7-6a3e-46ac-8376-85d9df951948, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.716726011Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.716758561Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ea30bac-13f2-4e1c-b902-1f3b630fdfdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.735166761Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.735203623Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: cc1d03a8-51c6-471a-9d2b-b6a143151eee, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.753709735Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.753750414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 096da1d3-d66d-49ff-8236-d77eb3631270, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.772516493Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.772548687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e40b6155-d379-4874-9119-51a498c972c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.791027772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.791057361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b130f689-a354-4ce5-be2b-20413d917375, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.809960333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.809987745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2394e76f-ffd7-4010-81b9-fd582588a6a2, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.82869455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.828720697Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13149332-2b9e-43f3-86ad-209fb5b70e55, dataset_name: DatasetName { pool_name: ZpoolName { id: a5a77acc-8f53-4c36-a482-41fc0a50e4bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.847042363Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.847071961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d8dfdb8-2c0d-4a97-a72d-a30868f01e39, dataset_name: DatasetName { pool_name: ZpoolName { id: 915c0c40-3159-4259-9f5f-ef7ec918e13c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.865309352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.865337666Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 89558561-0976-4295-a611-2d4619e6b7a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c590c44-e234-44db-965e-9f6e7745f594 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.883929821Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.883960599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 99f4fd41-b4a6-41ca-b42f-f4371f49bd8f, dataset_name: DatasetName { pool_name: ZpoolName { id: bb91d72d-293b-493d-a375-9a8656905e43 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.902349262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.902387775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b27d58f3-b052-4217-923a-62b3896e867a, dataset_name: DatasetName { pool_name: ZpoolName { id: fa33182c-5386-4788-bb69-fed5eea70499 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.920730244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.920763161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fc09e276-dc62-4ef0-a414-a788a80131ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 62e8a5f9-9c11-40b3-8243-b440c8ecad6f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.94016136Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.94019543Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 22fbbb08-eb93-44da-a6da-ac079d8c61cc, dataset_name: DatasetName { pool_name: ZpoolName { id: ebb69d76-d0be-447a-903a-9633ca2c2c1a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.959158203Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.959189544Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7d0cdafd-c4b9-4211-a037-354ab6337ccd, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb5d21f-d22a-413f-a49c-154a4b1cfaf9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.979481119Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:16.979517383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 153af4c7-3f5c-4702-aad3-4d1551a443cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 6063ad03-661e-4d00-b0bf-ebbe587f7c8b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.000258354Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.000291693Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.000492995Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.000607804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167152778Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167509455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167526987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167546749Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167575568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167587757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167593822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.167599672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.16873768Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.180701648Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.368640371Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.368876097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.368903444Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.394026816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.394196516Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.39422107Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.408178087Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.408302804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.40831006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.408368852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.412659894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.41680979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.4216011Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.426631299Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.431506639Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.43653838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.442058063Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.566193749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.571776653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.576847732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.581995488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.586981298Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.591792969Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.596368624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.601555173Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.606783021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.612530204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.618875835Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.624922955Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.630827588Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.636917521Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.68349315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.683825778Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.690347979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.696053618Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.702433116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.710118359Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.717378631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.780560441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.837489444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.906141842Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.910990956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.911005835Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.987957739Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 159, 221]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.987982621Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.991795669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:17.991808644Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.073789622Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 158, 118]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.074343674Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.075497813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.075514707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.168249757Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 235, 75]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.168272988Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.270087523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.272540493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.272553703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.325512227Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 208, 59]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.325542115Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.327024117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.327036773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.3797924Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 155, 3]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.379824454Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380054285Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380067654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380104021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380111727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380168672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380186747Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380776302Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380788255Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.380881031Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.382634698Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.38471913Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.384730164Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.38474854Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.386524571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.386617921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.386624138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.386730882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.485575753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.489230409Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.489262144Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.585812245Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.595541593Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.645050737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.65290208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.659950535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.666828964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.672413889Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.679967129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.68696291Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:18.70336462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.288379702Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.288595434Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.339091565Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.339304447Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.407552421Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.407807472Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.425919961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.427808251Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.486463478Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.486668756Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.490742309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.491031992Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.50452734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.50474609Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.540369155Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.540609553Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.549315725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:20.549604962Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 72.770527ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.71935774Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 28.584843ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.730529917Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 52.854186ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.742307491Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 50.456319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.753192938Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 39.740457ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.76168379Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 53.268269ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.769786318Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 63.980708ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.777009505Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 50.881816ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.785311389Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 25.824765ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.824794513Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 59.463184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.872955522Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 68.56192ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.892701797Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 51.824069ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.912141629Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 81.276346ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:20.977788318Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 74.783783ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.011283187Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 40.281233ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.032651259Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 284.138878ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.052059216Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 281.908576ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.071567541Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 290.995476ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.088647936Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 201.71993ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.107893587Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 136.177008ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.127837165Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 80.206927ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.207343051Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 84.485336ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.271492949Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 180.232135ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.399602011Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 105.037059ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.742934314Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 346.374719ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.793670633Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 221.779132ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.830146802Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 381.495907ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.862708877Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 326.691699ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.892209259Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 208.413319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.920890151Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 144.385838ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.943690379Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 206.470544ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.965770587Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 294.362789ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:21.986774005Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 122.864421ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.006737434Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 630.815703ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.076323896Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 339.606528ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.114543168Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 518.575984ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.15501859Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 373.577529ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.1779011Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 680.644809ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.202146289Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 427.699716ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.224414802Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 463.745611ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.259173646Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.156003533s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.283294291Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 589.53205ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.307680046Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.006657017s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.505035556Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 852.280097ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.571335923Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.64773839Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.648098714Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.151187778s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.671867738Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.672167555Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 705.650155ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.693627471Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.693839763Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 985.949239ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.732732318Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 673.585276ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.751192857Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.832689841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.833209702Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.048454467s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.901343591Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.421700747s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:22.916005564Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.954700431Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:22.955111489Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 56.232493ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.006030169Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 65.760125ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.041297806Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 108.765226ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.07872612Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 102.459538ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.160467519Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 33.586288ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.209926155Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 200.158565ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.266909015Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 125.913166ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.315666135Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 247.801237ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.362538526Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 72.046712ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.414367886Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.214322099s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.433577532Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 538.061513ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.452809249Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 744.716848ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.470234106Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 233.850959ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.483610346Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 736.737231ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.501894046Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 461.822521ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.520941764Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 902.54024ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.538695479Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 50.633256ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:23.55247648Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 154.673557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.088731667Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 865.505756ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.104980845Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 369.283012ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.120504505Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 802.958881ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.134380776Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.461991648s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.151047031Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 547.422485ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.164360857Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 521.104476ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.178852006Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 701.875877ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.191971109Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 736.156781ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.231276439Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 611.734804ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.254081782Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 218.478558ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.270817575Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.20207046s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.361213851Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.269667491s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.459425059Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.052769052s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.504931382Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.111836134s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.51796707Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 815.663187ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.666217669Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 634.270276ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.717088689Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 521.238339ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.730726503Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.295927277s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.88278231Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.205992612s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.910046385Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 816.955982ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.95531664Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.053802239s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:24.986962626Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.172466158s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.002020257Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 646.08204ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.26699148Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:25.281636754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:25.281833885Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.441710851s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.381882782Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 662.174686ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.508306894Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 986.333184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.577755594Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.11492255s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.594733146Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 38.27278ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.625764892Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 963.594667ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.641327355Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 775.301707ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.653561281Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 75.395523ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.675972854Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 806.372569ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.774237305Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 194.51718ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.851017294Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.469425446s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.886562934Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.433895148s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:25.934291662Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 972.658811ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.05914277Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 272.075233ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.077400196Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:26.084152392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:26.084611882Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 709.13969ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.134728534Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 836.272902ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.188074972Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 837.886148ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.204577549Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 965.236946ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.220244745Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 70.831407ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.360491225Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 657.38981ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.375436025Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 516.410911ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.448033939Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 81.248446ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.512000319Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.159642647s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.597530913Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 994.88421ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.62539408Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 147.147119ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.654130264Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.363180278s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.683010068Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.276888124s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.735037543Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 209.429485ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.819872052Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 706.096319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.847662223Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.130810881s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.864830319Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 964.434651ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:26.980830547Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 876.891158ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.061965782Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 710.879809ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.075219248Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 790.775246ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.087616129Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 460.953633ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.100731616Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.2184722s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.11520221Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 763.565146ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.202219652Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.004261177s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.376772295Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 805.946252ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.389666833Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 985.682374ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.570258039Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 807.563003ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.586696607Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.330681253s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.639925148Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 767.758919ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.780704865Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:27.78171685Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:27.782399085Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.209304168s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.814691209Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 858.738771ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.897424887Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 834.874037ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:27.958978657Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.401159877s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.004203798Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.409346762s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.089701936Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 998.848448ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.114822625Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.412931661s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.133746296Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.474056022s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.153219863Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.348170465s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.215777827Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 28.186665ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.310666575Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 604.851122ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.353933985Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 130.07645ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.379103738Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 832.890725ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.400408772Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 948.207004ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.419373195Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 156.120553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.523206943Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 965.535344ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.564611736Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.098664211s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.57692352Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 449.193262ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.692223923Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.003176894s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.774566822Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.002330907s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.810621456Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 825.409636ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.978224471Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 768.403497ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:28.995791696Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 727.576679ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.043631469Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 735.177147ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.130815705Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 529.633147ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.155646038Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 854.666266ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.250297491Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.07233279s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.383951783Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 869.793522ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.422461992Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.184223432s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.516381767Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.081519197s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.545811753Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 578.614772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.564603559Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 971.765553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.581215048Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.384659852s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.644721669Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.47830648s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.693700044Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 728.881939ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.70568391Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 855.175422ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.779860154Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.336540156s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.79317337Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.041764617s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.804253262Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.198193163s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.817326335Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 705.97583ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.827774922Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.213415739s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:29.882626486Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 675.618877ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.12135999Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 984.631899ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.158979814Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 973.629706ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.311304291Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 526.611213ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.454014532Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.390796895s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.475742524Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 707.913395ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.550229732Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 955.298973ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.57053574Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.097643143s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.645527812Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 750.976421ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.660963701Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.111360616s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.723538166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 606.553278ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.815158427Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.270681218s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:30.89121931Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.495146955s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.001752962Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.012973757Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.013016442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.013240573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.01326791Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.02257955Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 829.564057ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.043465468Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:31.043554278Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 999.41982ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.064705256Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.384196109s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.117152074Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 784.559665ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.15127166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.129100689s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.172317207Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 737.719429ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.196136201Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 817.761894ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.275960506Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 38.75688ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.289911965Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.077875835s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.302657226Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 31.185439ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.31416275Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 73.884768ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.326096355Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 58.463374ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.340508224Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 146.925255ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.357760428Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 141.639451ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.621189654Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 569.787591ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.646233467Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.18082282s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.667177581Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.175590984s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.683632088Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 213.670626ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.707403512Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 107.238216ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.72198723Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 662.911873ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.758834509Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 327.589095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.776125314Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 260.151422ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.842369318Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.01489169s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.852623612Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 928.786507ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.880400371Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.17393697s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.893355377Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 437.09418ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.932879893Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.300150616s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.948871047Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.237821224s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:31.960222547Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 674.734664ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.07915443Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 939.096669ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.10801862Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 907.592489ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.120053606Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 478.02305ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.13105739Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 992.179846ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.173858627Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.251353924s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.226755186Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 567.386844ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.319909Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 1.187959822s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.383797665Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.098106387s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.396322224Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 843.975285ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.436504468Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.33929989s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.509549498Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.43642867s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.517894773Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 487.585116ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.623309675Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 590.458615ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.770135138Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 773.884771ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.822707725Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 635.577891ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.859680054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 821.839641ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.871060043Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 916.907976ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.880697569Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.114252287s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:32.900585972Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 753.305109ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.040063991Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 731.67918ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.05743137Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.355112093s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.077648345Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 734.989901ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.121586344Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 783.754038ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.177668041Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 807.550566ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.211025311Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.391481961s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.264632609Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 520.813753ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.295338542Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.135126364s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.376367223Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 922.212711ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.493910104Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 763.632416ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.50965719Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 827.626544ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.521158508Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 1.087963385s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.585656613Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.483807573s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.610717192Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.488857149s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.703516483Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.251135506s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.803930371Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 828.587628ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.816101484Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 631.17098ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.827148123Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.213790556s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.838843524Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.388401887s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.861942134Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 513.173463ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.870613445Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.444616181s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.965338222Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 553.344828ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:33.973307374Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 893.300107ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.025467565Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 798.488964ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.033003711Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 580.344969ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.28489225Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 660.642406ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.360615768Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.498296145s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.394352371Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 558.900131ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.428275139Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 673.553557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.44578999Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 988.05702ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.46924181Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 714.139261ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.523236001Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.055297909s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.536350101Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 948.258328ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.655597303Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 772.710393ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.667086592Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 538.093594ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.683668416Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 779.057382ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.845015499Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.361308567s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.881382888Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 610.934366ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.933836863Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.350288336s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:34.999469112Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.4068305s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.034341408Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.44932747s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.065333506Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.357072659s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.075084896Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.042426718s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.115295556Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.484315383s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.135867473Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 706.387953ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.208708766Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 922.012057ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.235909055Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 834.485331ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.250755284Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.406911962s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.265588773Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.250297779s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.425277494Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 867.063551ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.456290913Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 1.191362108s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.474412932Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 980.122548ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.563403513Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 903.426303ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.60726552Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.13364574s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.621012142Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 715.776283ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.637709409Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.361614026s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.90830898Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 642.797801ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:35.931293664Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 610.514606ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.101580419Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 653.568563ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.172570994Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 988.36324ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.186577799Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.011374828s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.256771161Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 930.351557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.339013419Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.469290653s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.363968004Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 801.848698ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.378391786Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.178550046s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.447693324Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 918.673116ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.461929054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 600.28274ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.524128493Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:36.537847634Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:36.540640917Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"813523cf-8765-4417-8f98-07aaecb6feff","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51817","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1614,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 502.045685ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.546695621Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.067682577s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.558508333Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.099309919s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.58754055Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 891.71464ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.633731562Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 950.190894ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.679206813Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.423258883s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.689963399Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 726.177749ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.700539412Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.197257505s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.724370619Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 507.041841ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.767116888Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 801.938742ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:36.838823454Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.102920704s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.058705643Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.443977264s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.135815922Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 672.436281ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.184353274Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 946.143933ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.192932992Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 637.631882ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.278942728Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.149172271s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.289643703Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.32867142s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.298906576Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 581.920824ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.307888614Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.214574297s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.391141258Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 783.8184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.436969164Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 640.104891ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.535122157Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 577.336164ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.637263841Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 693.501403ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.645658248Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 1.3778028s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.653012984Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.468898544s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.659749527Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.280135156s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.697284546Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 986.325471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.844701908Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 737.878812ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.86753803Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 754.485644ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.900160643Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 581.955234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.925939701Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 1.177289411s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:37.934819947Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.266696348s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.125892433Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 1.109299582s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.150569333Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 1.225430157s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.172275998Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 550.103182ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.186130491Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.152329001s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.225054901Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 955.860569ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.234133741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 1.015576447s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.349491019Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.44704764s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.448223132Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d\") failed: Property not found. retry in 1.443493043s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.517831454Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375\") failed: Property not found. retry in 1.28368317s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.590157179Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 1.073580689s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.615881146Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 911.182379ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.625283771Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 622.050269ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.636702837Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.331062012s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.664210291Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f\") failed: Property not found. retry in 1.241853807s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.746275586Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 520.927432ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.841692237Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d\") failed: Property not found. retry in 1.109230395s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:38.989013057Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 885.461198ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.040588583Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc\") failed: Property not found. retry in 758.263319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.123628752Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 778.509174ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.13905063Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71\") failed: Property not found. retry in 1.032627674s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.199430481Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd\") failed: Property not found. retry in 1.325748025s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.268540516Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55\") failed: Property not found. retry in 544.915805ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.277351902Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a\") failed: Property not found. retry in 1.474477761s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.37521586Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39\") failed: Property not found. retry in 600.993813ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.384333881Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5\") failed: Property not found. retry in 1.177570219s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.39334958Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 510.768463ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.407551579Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf\") failed: Property not found. retry in 515.068401ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.419362341Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 977.538303ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.547319094Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1\") failed: Property not found. retry in 915.999846ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.701617301Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad\") failed: Property not found. retry in 1.16052586s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.955469105Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 1.36766505s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.966773599Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 797.998194ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.977085608Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 740.201325ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:39.986544973Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.441660978s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:40.160523885Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 1.282823857s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:40.535476845Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 810.427461ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:40.738603776Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.269849043s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:40.786636826Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 955.975098ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:41.344298593Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 899.19997ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:41.559260806Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.19325351s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:41.614966435Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 560.839547ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:41.832342655Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2\") failed: Property not found. retry in 1.265664399s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:42.080638812Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 1.036731493s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:42.323378134Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 926.109752ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:42.417762779Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 712.006318ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:42.481648684Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.362067438s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:42.832267399Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.479315792s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:43.21767588Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 1.494716252s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:43.365488828Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 1.475985044s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:43.385169072Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.319317805s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:44.21477428Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1\") failed: Property not found. retry in 1.382312495s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:44.710850986Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 666.394965ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:44.876436418Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 826.509441ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:44.890014177Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076\") failed: Property not found. retry in 1.095964762s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:45.547856798Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6\") failed: Property not found. retry in 528.087557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:45.558515542Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8\") failed: Property not found. retry in 880.77973ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:45.729484998Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T18:54:46.132545711Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"57.624852449s"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671180553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_13149332-2b9e-43f3-86ad-209fb5b70e55"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671234604Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_oximeter_d3774998-58a6-4b89-8d53-ad0f914d458d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671239984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671244304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671248814Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_80ad703a-fbb4-4254-a4ab-a4358356c4a5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671253204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_7d8dfdb8-2c0d-4a97-a72d-a30868f01e39"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671257654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_89558561-0976-4295-a611-2d4619e6b7a1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671261784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_b130f689-a354-4ce5-be2b-20413d917375"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671266194Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_01130983-13be-453f-8cd5-3f2c8481c6c8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.67166672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_clickhouse_2394e76f-ffd7-4010-81b9-fd582588a6a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671672651Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_80a8c73c-5c3b-49f3-8129-27ed1a596c1d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.67167918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671684721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_e40b6155-d379-4874-9119-51a498c972c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671691851Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_153af4c7-3f5c-4702-aad3-4d1551a443cf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671696481Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_22fbbb08-eb93-44da-a6da-ac079d8c61cc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671707561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_b33ec370-f410-4564-a49d-41d565d13ca6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671726721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_99f4fd41-b4a6-41ca-b42f-f4371f49bd8f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671731431Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_434fbf80-f2f8-48ff-9cc8-d0faa5cd0a71"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.671735561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_51c1446e-49e7-4598-8832-f33ccdf3c076"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.672653075Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.67552242Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.677058054Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1e88546b-45a5-49af-b101-2c695b98d6f5","remote_addr":"[fd00:1122:3344:101::1]:40024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30022348,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.685842009Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1321"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.686040272Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.686164404Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.689051528Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.689314222Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:719"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.723043402Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:886"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.725013023Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.785436964Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:46.862839367Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:47.006119966Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:47.211518282Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:47.462226787Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:47.727860982Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:48.122284152Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:48.432000366Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:48.972557048Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:49.243937572Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:49.831307566Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:50.310082697Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:51.275745972Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:52.827865267Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:53.808093928Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/d6d74c37-fa13-4dad-8a2e-066574b53674/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:56.106135382Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0752573a-f744-4256-89b8-af4f65504527\", \"content-length\": \"173\", \"date\": \"Thu, 24 Oct 2024 18:54:57 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"d6d74c37-fa13-4dad-8a2e-066574b53674\\\"\", request_id: \"0752573a-f744-4256-89b8-af4f65504527\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:54:57.597375159Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.302261584Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.387733022Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33038"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.388188889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.388212839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.388233759Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.389280806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:00.624552792Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"6f7053e8-a669-4ef2-932d-969504b688be","remote_addr":"[fd00:1122:3344:101::b]:33038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":236708,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.735553259Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49573"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.739434978Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.739621821Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740691408Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740705188Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740726128Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740741958Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740758679Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.740785009Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6996d0da-402d-4761-bae9-6795d0d30d7f","remote_addr":"[fd00:1122:3344:101::c]:49573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5130,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:01.742713339Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.147201004Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58532"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.147958566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.147966586Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/1efc7c65-a878-483c-82d1-872a03292008/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.147995236Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.149118824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.36229552Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"6c22a362-cdb0-4974-ab81-600dfdaee3f3","remote_addr":"[fd00:1122:3344:101::c]:58532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":214997,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459416307Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36055"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459655451Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459693801Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459730452Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459754232Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459766462Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459772483Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459794443Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.459802823Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d4b47307-80f1-4f27-bba5-de39bd2cd158","remote_addr":"[fd00:1122:3344:101::b]:36055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":324,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493305889Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58427"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493497332Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493534623Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493568084Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493578934Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493595754Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493606834Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493619724Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.493647725Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c7dae446-4d5c-490d-935e-34a0a5e5db58","remote_addr":"[fd00:1122:3344:101::c]:58427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.697320824Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:32811"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.697559338Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.697629799Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.69767144Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.69769204Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.69769885Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.69771618Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.697724991Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:02.697754641Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d8e86b97-618c-4ece-b8ca-0fb20add7064","remote_addr":"[fd00:1122:3344:101::a]:32811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":342,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:06.578875978Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:06.57899855Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"fd330ab2-fffa-47ad-9661-544248982459","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54121","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:07.977827773Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:07.978226679Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"037cd90b-33c5-41b8-a319-f2e446e46e42","remote_addr":"[fd00:1122:3344:101::c]:41412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":339,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:08.021248692Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:08.021418025Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e7c1b063-e842-4692-bf60-3434afe3bb0d","remote_addr":"[fd00:1122:3344:101::c]:41412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":463,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:08.111966721Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6a140745-ef80-4e80-b810-142d8a93a2bd","remote_addr":"[fd00:1122:3344:101::c]:41412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2320,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.735131438Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40879"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.735290451Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e5509c1d-3626-47cc-8115-ff5303d00827","remote_addr":"[fd00:1122:3344:101::b]:40879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.786143585Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.786166435Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8f721bad-35bb-46c1-a860-0aae375b2579","remote_addr":"[fd00:1122:3344:101::b]:40879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.863353305Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51728"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.863509747Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2e122b83-ef77-4d7d-8860-c64b3cd180f8","remote_addr":"[fd00:1122:3344:101::c]:51728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.88834508Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47110"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.888859458Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cd856160-1f8a-4499-bb63-15556ffdc9fd","remote_addr":"[fd00:1122:3344:101::b]:47110","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":384,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.898025269Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"24e19870-c299-4afa-badb-b99f24f2a11f","remote_addr":"[fd00:1122:3344:101::b]:40879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":424,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.915308086Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.915326156Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"411fc7fe-9028-45a9-bd0c-251738fc0cdd","remote_addr":"[fd00:1122:3344:101::c]:51728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.991621178Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.991910194Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"81161ffe-983d-4aae-9a68-456ceaaabd3e","remote_addr":"[fd00:1122:3344:101::c]:41185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:28.994546477Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0a63f7ef-eca1-49ab-949a-638b5418d8d9","remote_addr":"[fd00:1122:3344:101::c]:51728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":295,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.001048737Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.001128299Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cc960c5b-c512-4bff-8c65-2cb9e8219e6f","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.006929598Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58994"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.007522757Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.074415679Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.138490296Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"709345f7-69f4-46f2-96db-a61797f82218","remote_addr":"[fd00:1122:3344:101::b]:58994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131509,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.13872334Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34708"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.13877484Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.138830912Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"968bdd20-cde9-4ab9-bff3-26cd82946da5","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.139097585Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"762fffef-239e-4b34-b1f4-7af98324c228","remote_addr":"[fd00:1122:3344:101::a]:34708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":205,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.140242153Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"001eaf8d-1047-4b92-b4c1-771cb3e8f41f","remote_addr":"[fd00:1122:3344:101::b]:58994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1463,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.148762185Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a179f337-57e4-4c88-ac5b-c2c6e4aa0184","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":246,"response_code":"204"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.195557826Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/1efc7c65-a878-483c-82d1-872a03292008/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.196056594Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/982c7515-f8a4-4d54-b007-05451b94e502/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.198149186Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.199517787Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.199530987Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"93bc57ad-604a-4ba0-ae80-2928ac6a8e1c","remote_addr":"[fd00:1122:3344:101::b]:58994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4398,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.213406591Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38988"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.213535783Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.256288092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.309006635Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"65348a2e-d0c5-4478-8170-0458e70f2794","remote_addr":"[fd00:1122:3344:101::a]:38988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95529,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.309159897Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44958"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.309402091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.31259891Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"264c57cd-78a9-476f-a8b3-9e7486e7f4fc","remote_addr":"[fd00:1122:3344:101::a]:38988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3277,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.343291283Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.395244224Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"62511bc5-254e-4548-b2a3-2e9ea20b26f8","remote_addr":"[fd00:1122:3344:101::c]:44958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85926,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:29.39562935Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"10b7192a-9f69-44bf-ad1f-2b028940ab1c","remote_addr":"[fd00:1122:3344:101::c]:44958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:36.616687929Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:36.616831561Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"24fba5b6-1731-47f4-89dd-dfa8ef189636","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59049","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.707415229Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.707639472Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"52d886a4-f348-4a8a-9dc1-fefa50113088","remote_addr":"[fd00:1122:3344:101::b]:48487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.7866247Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.786688871Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5a5984b9-dbce-405b-ac5f-9f3cf778361e","remote_addr":"[fd00:1122:3344:101::b]:48487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.800661446Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cd4a1d65-86a3-4305-bae3-806ffee13912","remote_addr":"[fd00:1122:3344:101::b]:48487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.946236921Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58791"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:58.946356792Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8dc18b22-b479-4e18-b8b0-d1a9b714c68e","remote_addr":"[fd00:1122:3344:101::c]:58791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.012202437Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.012233098Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c5842a5d-af03-43a3-9edc-797ffb7bd33d","remote_addr":"[fd00:1122:3344:101::c]:58791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.023623193Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"371da416-c718-416b-8368-298e82ebf592","remote_addr":"[fd00:1122:3344:101::c]:58791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.083387525Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.083573598Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"0c8d9b80-83d8-417f-a93c-32f0bad318d1","remote_addr":"[fd00:1122:3344:101::c]:44151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.102494799Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.102638812Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d48a0fc9-c819-4b89-aaf6-a4ba25d6bbd4","remote_addr":"[fd00:1122:3344:101::b]:36888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.10642218Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:65445"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.106510511Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.144888813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.201046768Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"14b17815-15d9-4fe7-a54f-45aa8a217409","remote_addr":"[fd00:1122:3344:101::c]:65445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94574,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.201343583Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"44cff308-eda2-4686-be43-4e128abc6bbf","remote_addr":"[fd00:1122:3344:101::c]:65445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.234736968Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53186"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.23489002Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.27576092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.330531014Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3c00b5c5-1527-48ce-9341-78b53f13936b","remote_addr":"[fd00:1122:3344:101::b]:53186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95706,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.330676927Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39649"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.330811469Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fcac069b-c2e6-468c-9171-d43362d70b86","remote_addr":"[fd00:1122:3344:101::a]:39649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.33091975Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a4b850b5-99ea-4d15-ac62-20388e76fba3","remote_addr":"[fd00:1122:3344:101::b]:53186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.387678835Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59388"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.387885109Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8fe593f2-a1ed-484d-9cdb-ccac8de7f0b4","remote_addr":"[fd00:1122:3344:101::a]:59388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.451209645Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.451231475Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cc6622ca-4c08-42aa-a600-52fc581cef7a","remote_addr":"[fd00:1122:3344:101::a]:39649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.455532141Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52683"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.455608942Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.493492446Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.546013256Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"aa3cf162-f67a-4e66-a64e-26572df78f50","remote_addr":"[fd00:1122:3344:101::a]:52683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90437,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.54629613Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c19fdae9-3b3b-4585-8c0f-f5af43f871d1","remote_addr":"[fd00:1122:3344:101::a]:39649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":184,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:55:59.546423302Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"17f4f5da-4be7-426b-a613-bc4ea29bb265","remote_addr":"[fd00:1122:3344:101::a]:52683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:06.657236571Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:06.657338593Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f5ecff84-d0ad-4cc3-a700-7929ac6cdee5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53525","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:13.901445856Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:51556"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:13.954485664Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"ba13df5d-b071-49e8-b52c-67fc76c96543","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52938,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.720648907Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.720929832Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d3d3abb3-d563-42f3-96ce-37a315b71cd5","remote_addr":"[fd00:1122:3344:101::b]:41647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.765755153Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.765882665Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"df874ca6-2d2f-4561-b5f3-bc9bb63e8708","remote_addr":"[fd00:1122:3344:101::b]:40744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.810746056Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.810779407Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3129f861-409f-414a-801a-9318e996694d","remote_addr":"[fd00:1122:3344:101::b]:40744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.822090671Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4184de98-7c19-40fe-a6cb-5b123e2e76dc","remote_addr":"[fd00:1122:3344:101::b]:40744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":229,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.865799555Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.865961187Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"baea3689-897c-44a3-86e5-11b8d4ec1eb5","remote_addr":"[fd00:1122:3344:101::c]:47305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.886600116Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.886619626Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ac40981e-db76-4118-9425-a18546504991","remote_addr":"[fd00:1122:3344:101::c]:47305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.897871929Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a6f2b26d-e2dc-4b46-a553-f3cfa088fbe8","remote_addr":"[fd00:1122:3344:101::c]:47305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.948695803Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42668"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.948805384Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0aadb517-6207-4b22-85b5-58342505d4a1","remote_addr":"[fd00:1122:3344:101::a]:42668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.955522448Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50729"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:28.9556661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.006643066Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.075819722Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"43d7e0d3-9081-4c29-8f4a-8e51534a1b6c","remote_addr":"[fd00:1122:3344:101::b]:50729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120206,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076079976Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076101547Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076110947Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9e143c1c-fbb4-4509-b608-5f7677cc6623","remote_addr":"[fd00:1122:3344:101::a]:42668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076155998Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61148"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076416651Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4b3cd4a7-2973-46a8-8ec2-cbbfd1fa8482","remote_addr":"[fd00:1122:3344:101::b]:50729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":286,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076502403Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2578bb08-507f-438a-bcc2-c189adee7e65","remote_addr":"[fd00:1122:3344:101::c]:59049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.076516413Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.087709046Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"910c5615-b7ce-467f-957a-aec25bb15f4c","remote_addr":"[fd00:1122:3344:101::a]:42668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":408,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.115777268Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41428"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.118238946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.13533956Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.135630344Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"4ab020cc-002c-4940-99b5-5487621e00e2","remote_addr":"[fd00:1122:3344:101::a]:44547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.177056266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.17712402Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fd2bd065-c3c4-41ac-a242-2190d7e46460","remote_addr":"[fd00:1122:3344:101::c]:61148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100746,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.182767671Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3d96eae2-4e10-4775-b1af-baf76755b7b8","remote_addr":"[fd00:1122:3344:101::c]:61148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":747,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.220655045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.283717637Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3deefbcc-b5e6-47e3-8f21-cf62c3309b7b","remote_addr":"[fd00:1122:3344:101::a]:41428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167790,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:29.284076553Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7175d73d-fa3b-4336-80ed-58745d9f63ec","remote_addr":"[fd00:1122:3344:101::a]:41428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:31.301737744Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:36.696063054Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60436"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:36.696165605Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"6bbf94ea-48f0-4e8f-baec-c9227bbe9444","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60436","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:43.909977612Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"098915cc-b9f2-4fa3-9e88-1ba1adbc5cb2","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8192,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.869616766Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49903"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.869786329Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"69fa8ec9-5210-4843-a14c-b35a61830427","remote_addr":"[fd00:1122:3344:101::b]:49903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.942456419Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.942488419Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8e98b506-2692-406e-90e2-21156562ff93","remote_addr":"[fd00:1122:3344:101::b]:49903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.953830424Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"895394c1-185a-48b1-92be-3e0204b4afe7","remote_addr":"[fd00:1122:3344:101::b]:49903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":231,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.976359381Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.976453713Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"05c01afa-c50e-408b-bd6d-89ca9404c8b1","remote_addr":"[fd00:1122:3344:101::c]:56587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.992258116Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:58.992281807Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c0438ddc-7734-4d23-b4ea-1522277bffe4","remote_addr":"[fd00:1122:3344:101::c]:56587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.005782495Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0f61ae58-7aa6-4562-bfd1-cb1bd0a6e825","remote_addr":"[fd00:1122:3344:101::c]:56587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":232,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.145458848Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.145660571Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"71476525-798d-4ef6-b0df-f9d3efecb072","remote_addr":"[fd00:1122:3344:101::c]:36357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.14887872Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.148989712Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b169ebc0-9990-40d1-8831-df88703f8a61","remote_addr":"[fd00:1122:3344:101::b]:53242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.163503006Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58156"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.163617178Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.213017259Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.287405046Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8b6902a2-1db2-4775-992e-136c1e1687b6","remote_addr":"[fd00:1122:3344:101::b]:58156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123830,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.287508657Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43466"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.287540498Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36061"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.287715561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.29286217Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"32dcc497-c8cf-43b2-b4aa-0fd01654eb2c","remote_addr":"[fd00:1122:3344:101::a]:43466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5017,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.292997782Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c2cd29ea-4977-4afc-aa3e-27599bc93e40","remote_addr":"[fd00:1122:3344:101::b]:58156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.323687275Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.32397243Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a8606539-c68d-4372-b980-ba5d43a665ad","remote_addr":"[fd00:1122:3344:101::a]:42160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.32529757Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44130"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.329332332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.336258819Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.33631237Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1ba3d950-9586-48fe-a226-a6f52dc7a680","remote_addr":"[fd00:1122:3344:101::a]:43466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.347776426Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1f4208d5-4aa2-4950-94b0-a8adc7b94a50","remote_addr":"[fd00:1122:3344:101::a]:43466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":905,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.395451741Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.395520142Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0b55f5d1-e5bf-4d99-aa07-a157d2963fc3","remote_addr":"[fd00:1122:3344:101::c]:36061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107860,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.399340981Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e7a77a7a-a065-49b4-8b39-672d4dbc4d3c","remote_addr":"[fd00:1122:3344:101::c]:36061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":299,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.432431471Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.489147456Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a2e537db-ae28-42e4-966a-422e4d19fa99","remote_addr":"[fd00:1122:3344:101::a]:44130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":163780,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:56:59.489495641Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8df16a31-b0a1-43bc-a8ea-5e4e6418d3f9","remote_addr":"[fd00:1122:3344:101::a]:44130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:06.735862509Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:06.735962671Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"872f1b0c-bfc5-4e14-ad5e-c5724dff2a77","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56765","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:13.910128378Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"8e437ffb-b421-4f77-a33b-296646f90a1c","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7201,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.659885827Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.660031389Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1c7fb8f4-af02-4d57-b667-6e254a22e674","remote_addr":"[fd00:1122:3344:101::b]:62121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.724428952Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.724497603Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c403f545-0020-4fa2-af23-ce56ba83b80b","remote_addr":"[fd00:1122:3344:101::b]:62121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.772992251Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.773187384Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a71ca36f-5840-4d6a-915c-a48a99e3f063","remote_addr":"[fd00:1122:3344:101::b]:47127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.776534385Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"41f444ea-e173-4aa1-b9bc-4b83b8f56129","remote_addr":"[fd00:1122:3344:101::b]:62121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.915864913Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35243"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.916003665Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:28.959631128Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.020733799Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7141ced4-7bd9-4451-9fe1-5b6a990d851d","remote_addr":"[fd00:1122:3344:101::b]:35243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104790,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.021094175Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"77095396-ca2b-4fc1-b8fa-a9adb27a513b","remote_addr":"[fd00:1122:3344:101::b]:35243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.075307841Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.075462383Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2a05b7ea-673c-4307-b0b2-1b777024a3c5","remote_addr":"[fd00:1122:3344:101::a]:47317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.114536865Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.114561146Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5c5b4aba-7947-4c3b-8cb4-bfcfe4cdd7dc","remote_addr":"[fd00:1122:3344:101::a]:47317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.152398449Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.152598272Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8e984ed9-7034-42a8-bf2b-4f19e7408d52","remote_addr":"[fd00:1122:3344:101::c]:39599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.154877977Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.154965699Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c3edb1db-392a-4d70-9a59-4cbb0569950c","remote_addr":"[fd00:1122:3344:101::c]:47835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.189497881Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57678"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.189603863Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.235663153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.303304365Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bd234d49-f577-47b1-95b1-31cc66703158","remote_addr":"[fd00:1122:3344:101::c]:57678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113729,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.30360848Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53282"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.303709041Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.303753502Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6b7f15f2-e9e3-47bd-ade0-d3b0409f40ae","remote_addr":"[fd00:1122:3344:101::c]:47835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":223,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.303949885Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"51556356-c531-4004-b3a3-4f1438fb6505","remote_addr":"[fd00:1122:3344:101::c]:57678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":269,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.304052097Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"f825aec6-2100-45ed-b0de-cfd7131af1e9","remote_addr":"[fd00:1122:3344:101::a]:53282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":269,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.304190899Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"141e5f99-841b-4642-a5a0-398da87c53ae","remote_addr":"[fd00:1122:3344:101::a]:47317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":457,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.30880246Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44095"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.308887531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.346376889Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.402820979Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d21c8a2c-975d-43ed-af32-955c7b279bfc","remote_addr":"[fd00:1122:3344:101::a]:44095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93971,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.403287876Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d7faaa4e-aeab-4600-8289-5ab91bef322e","remote_addr":"[fd00:1122:3344:101::a]:44095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":234,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:29.403316727Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"734f8994-4c0c-4e08-a818-1be23b6a9e76","remote_addr":"[fd00:1122:3344:101::c]:47835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":304,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:36.772106014Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:36.772204636Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b891a8f9-1d27-4824-830f-077f97abd1b1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46888","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:43.911255041Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"af9b189b-3190-4d00-ac85-b2315e89df8e","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7812,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:58.875069212Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37845"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:58.875311246Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bd6a029c-66ea-4d5e-8f85-f5d84680ee0b","remote_addr":"[fd00:1122:3344:101::b]:37845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:58.904265732Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:58.904295413Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2ef0d355-63b4-420e-85a8-4130217fc928","remote_addr":"[fd00:1122:3344:101::b]:37845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:58.915196531Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9aa485fb-71fc-4f3e-935f-a00262bc516e","remote_addr":"[fd00:1122:3344:101::b]:37845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.043767412Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.043937515Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"42e4eb5e-adf6-4bc9-910e-bb6289ae85e8","remote_addr":"[fd00:1122:3344:101::b]:63399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.110588702Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.110679824Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"07a8d346-cd91-41c7-814d-e3378b8c11c4","remote_addr":"[fd00:1122:3344:101::c]:41161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.114234239Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.114381491Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"06bd6dac-c295-4243-b5ed-3f521e502b23","remote_addr":"[fd00:1122:3344:101::c]:49581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.119625832Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.119721343Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2e610f4b-5a09-4d52-ae03-0ba1af0b4e5e","remote_addr":"[fd00:1122:3344:101::a]:51450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.120726399Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34646"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.120845361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.167119014Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.233617099Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6d31f2d5-c1fc-4e57-9fc5-f8d4d34a4f36","remote_addr":"[fd00:1122:3344:101::b]:34646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112819,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.233780621Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35389"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.234205938Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.234240739Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ef832500-a029-4361-9895-9fa8aec4fc2a","remote_addr":"[fd00:1122:3344:101::a]:35389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":265,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.234249069Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.234264439Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f6da267c-29d4-47cd-8e20-ff4ee53142fb","remote_addr":"[fd00:1122:3344:101::a]:51450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":260,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.234281269Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"40c3954e-0d4d-43b3-8f43-524594b798b9","remote_addr":"[fd00:1122:3344:101::c]:41161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":292,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.23435067Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3f6a7a52-55c8-4402-aa7f-7f7ce932cfc8","remote_addr":"[fd00:1122:3344:101::b]:34646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":336,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.250750563Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"235d0a44-928e-4f0e-a74c-dfbf664b3fb5","remote_addr":"[fd00:1122:3344:101::c]:41161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":419,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.251310192Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fff28ef6-2d2b-4d13-9f17-61d3e910c6cf","remote_addr":"[fd00:1122:3344:101::a]:51450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":195,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.251714968Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44124"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.251798939Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.292635829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.349322693Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dde47ac1-99c4-4343-8a50-6957d9b29b31","remote_addr":"[fd00:1122:3344:101::c]:44124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97547,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.349639887Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.34981341Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a539ee31-d07a-40c6-bc0a-0f4e3e3d6d2d","remote_addr":"[fd00:1122:3344:101::c]:44124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.349923532Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.392975775Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.45102978Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e03d9b8d-8f4d-4ce2-9eda-644299e957f3","remote_addr":"[fd00:1122:3344:101::a]:41751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:57:59.451361495Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5672178f-03bc-49a8-8c47-6f016e09d212","remote_addr":"[fd00:1122:3344:101::a]:41751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:06.805282942Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:06.805373343Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ee5e0df2-d3f0-42c2-80f3-29c778bc17ec","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58690","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:13.917998951Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"42c468c1-fa53-4b07-96b4-6785cce631f7","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7374,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.655541254Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.655706277Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a16918d3-6d86-4986-a2e4-88e0d898e315","remote_addr":"[fd00:1122:3344:101::b]:45539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.713457576Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.713489836Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f5b27aaa-1bb8-4a13-9d53-11dfe1c0ab16","remote_addr":"[fd00:1122:3344:101::b]:45539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":164,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.723879137Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"49e5582e-0ff6-4e64-b104-445bded4a0e1","remote_addr":"[fd00:1122:3344:101::b]:45539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.757688076Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.757773428Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"39f12da5-d418-49d0-8c5d-1ede09ec4da5","remote_addr":"[fd00:1122:3344:101::c]:49729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.803608986Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.803635286Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"85112863-50b1-46eb-a82c-692d0c18c44d","remote_addr":"[fd00:1122:3344:101::c]:49729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":50,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.812513893Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a6f6ca1c-6a91-4f8a-8a11-83d5d38d90df","remote_addr":"[fd00:1122:3344:101::c]:49729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.911728442Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.911965046Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b06f52fb-3174-41e9-a855-5dc6b4b2f119","remote_addr":"[fd00:1122:3344:101::b]:50175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.927390974Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44948"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.927513996Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:28.969182548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031113743Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a13c28ad-53a2-4f49-9d65-b47ec8f72d8e","remote_addr":"[fd00:1122:3344:101::b]:44948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103636,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031268505Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40721"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031308006Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64115"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031368187Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.03157688Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c1e5c627-bba8-4a92-af36-b2510c5d3122","remote_addr":"[fd00:1122:3344:101::a]:43796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":124,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031740102Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c26b7d83-58cc-4bda-8e92-6a0d00ba8471","remote_addr":"[fd00:1122:3344:101::c]:40721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":240,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031753262Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a4916f0c-dd62-479b-abf4-2fdcd3e2dbee","remote_addr":"[fd00:1122:3344:101::b]:44948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":341,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.031768513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.075157051Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.080743668Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41023"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.080980781Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"6ec4ff0e-d30b-479b-8d23-8e8cd1203f32","remote_addr":"[fd00:1122:3344:101::a]:41023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":172,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.097799211Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.097851161Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cd28569e-25cd-4e57-95fe-01e49fec222f","remote_addr":"[fd00:1122:3344:101::a]:43796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.106783899Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a017dd42-2bf3-47c9-9f78-9900eed67235","remote_addr":"[fd00:1122:3344:101::a]:43796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":380,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.119980973Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64914"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.138674281Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.138766402Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5aff2564-f2f3-4fa2-a88f-94024cfd7ef0","remote_addr":"[fd00:1122:3344:101::c]:64115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.143251001Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6df6802e-7de2-4524-9dc2-a860cc33b7a4","remote_addr":"[fd00:1122:3344:101::c]:64115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":344,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.175099822Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.23207703Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e44a0219-467a-4935-ae63-e0e25202acca","remote_addr":"[fd00:1122:3344:101::a]:64914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111954,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:29.232427046Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2e628d67-f371-42b5-8808-02c519c33752","remote_addr":"[fd00:1122:3344:101::a]:64914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:36.84440727Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:32895"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:36.844503911Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ee0fc857-61b1-4756-9122-0f2407181a83","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:32895","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:43.991967606Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"275f413a-00af-4fc1-aa37-b0a25468277d","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8956,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.878756228Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.879139354Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a68d0a26-bb39-4999-b426-02642e00aeee","remote_addr":"[fd00:1122:3344:101::c]:46838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.887545053Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48488"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.887734366Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"054a6c21-24f6-498d-a207-de551a3e69d1","remote_addr":"[fd00:1122:3344:101::b]:48488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.974308001Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.974372752Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"610d5b65-5aaa-4f24-b1e6-bd68ef599383","remote_addr":"[fd00:1122:3344:101::c]:46838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.976748318Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:58.976768408Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"11c8ce60-271e-4448-994f-0cb4f6e9e77f","remote_addr":"[fd00:1122:3344:101::b]:48488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.004641478Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2b084d44-5976-40f4-b7c3-ab7fc18bca8b","remote_addr":"[fd00:1122:3344:101::c]:46838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":302,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.005212507Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a6810298-1402-43d4-97f1-1aa64a9b6798","remote_addr":"[fd00:1122:3344:101::b]:48488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.064206196Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.06445768Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cc27ed7d-a9d9-4fae-8bfc-561e38ab8fda","remote_addr":"[fd00:1122:3344:101::b]:47569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.078153671Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51841"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.078251913Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.122281252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.176825592Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"827a5735-7c13-41a5-b046-a69ac030d212","remote_addr":"[fd00:1122:3344:101::b]:51841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98608,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.177134867Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0614910d-6ba8-460c-8142-9ce8e727ba50","remote_addr":"[fd00:1122:3344:101::b]:51841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.185853282Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64132"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.186047505Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"95ba9975-4c14-468f-a9a8-d1b30df6f3cf","remote_addr":"[fd00:1122:3344:101::a]:64132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.205484624Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57890"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.205710578Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"83a689eb-1d44-491f-8350-b450d22e968f","remote_addr":"[fd00:1122:3344:101::c]:57890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.208513021Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.208528391Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c0ac8194-77e2-4696-82df-435c3268a13f","remote_addr":"[fd00:1122:3344:101::a]:64132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.274918944Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39927"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.275025456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.313827664Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.370848263Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fcee657b-6e02-440d-a88d-018d12479783","remote_addr":"[fd00:1122:3344:101::c]:39927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95863,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.371038746Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.371343271Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"51c20399-07f1-434c-8f14-4691e1b21f05","remote_addr":"[fd00:1122:3344:101::c]:39927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.371502983Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b94168ed-ecb5-4371-9786-56495f57b2bd","remote_addr":"[fd00:1122:3344:101::a]:51863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":257,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.371643895Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e8e3121e-8f13-4cea-82fb-2164043825b1","remote_addr":"[fd00:1122:3344:101::a]:64132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":448,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.381465927Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35651"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.381593989Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.420731642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.480600165Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9d1a8503-f49a-4110-aa7a-ebaa88a632c4","remote_addr":"[fd00:1122:3344:101::a]:35651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99052,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:58:59.48090882Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4c37b26e-5632-44cb-922d-5fa31efc555a","remote_addr":"[fd00:1122:3344:101::a]:35651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:06.878051063Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:06.878191195Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"feca9723-758d-435e-8988-72a26e273cad","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44619","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:13.990662881Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"edc29afa-b671-4edc-8d52-665676b06f59","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7597,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.790380122Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.790544094Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2a353474-8150-4918-9b07-cd348eed54f2","remote_addr":"[fd00:1122:3344:101::b]:46068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.807992053Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.808014383Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"443b5213-1785-4c80-be47-e08aabb59384","remote_addr":"[fd00:1122:3344:101::b]:46068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.823115756Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bd0883c3-c8c8-4040-88d3-3de2d57b1bf8","remote_addr":"[fd00:1122:3344:101::b]:46068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":282,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.895753566Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.895868837Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"388fc350-4911-4268-9397-abb26f331d46","remote_addr":"[fd00:1122:3344:101::c]:36187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.917113145Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.917148436Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"981cf7aa-beb5-4881-9e47-6c63cba36366","remote_addr":"[fd00:1122:3344:101::c]:36187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.934534914Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c397c4ee-e08c-4f05-875c-7eb999ada2f7","remote_addr":"[fd00:1122:3344:101::c]:36187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":301,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.934724287Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37627"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.9349296Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"039eed14-1f9c-4def-9960-47f311dbce4a","remote_addr":"[fd00:1122:3344:101::b]:37627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.941038744Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56749"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.941132745Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:28.991586543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.073497106Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e33ba381-1e01-414b-a2d6-849af7eead9f","remote_addr":"[fd00:1122:3344:101::b]:56749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132393,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.073637028Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65316"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.073665658Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41791"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.073986113Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b195c717-8b6a-4741-a091-33b2a9448bd1","remote_addr":"[fd00:1122:3344:101::c]:41791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.074158066Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7b164847-2bd3-4a31-b892-6769b89c391c","remote_addr":"[fd00:1122:3344:101::a]:65316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":224,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.074220597Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4df5ac19-a3ee-4c91-a411-8d915a15c584","remote_addr":"[fd00:1122:3344:101::b]:56749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":320,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.128445043Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41241"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.128567255Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.18601591Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.24048928Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dbd721d2-392a-4993-84ff-a40970f33f79","remote_addr":"[fd00:1122:3344:101::c]:41241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111979,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.240725313Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41199"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.240769704Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.240807955Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"eb433200-ed4b-4f63-b178-2f4fcd7fdb89","remote_addr":"[fd00:1122:3344:101::a]:65316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.240824725Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38464"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.240989408Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.241031748Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"78c82ec0-3518-42a2-9661-8c2ed6b902c1","remote_addr":"[fd00:1122:3344:101::c]:41241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":273,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.24634326Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5f37a59c-20b4-41bd-9b0c-f23c279f9e15","remote_addr":"[fd00:1122:3344:101::a]:41199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5297,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.251101733Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f2083224-7c8d-42a2-96de-8460b60fd27e","remote_addr":"[fd00:1122:3344:101::a]:65316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":305,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.281961729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.338763725Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"16b013b6-e4f6-4a44-ae3f-9ffc2e0302ac","remote_addr":"[fd00:1122:3344:101::a]:38464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97819,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:29.33913221Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"30f1e148-6449-4fa9-99ee-aaa4c9db0fd2","remote_addr":"[fd00:1122:3344:101::a]:38464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:36.911455724Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60535"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:36.911591406Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"079f5baa-2f35-48d7-9e54-e7aead3c4b82","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60535","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:43.991471028Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"6d87a2c1-a7cd-4709-82fb-23105acb87ca","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8090,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.798551404Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.798750407Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0bfcd8f3-747a-401d-93b6-2764eda24b59","remote_addr":"[fd00:1122:3344:101::c]:43185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.886595761Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.886635561Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cedc403f-d566-49f6-88ca-1948478bb7aa","remote_addr":"[fd00:1122:3344:101::c]:43185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.915402865Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b847e729-781c-4910-93fa-0bcc624a2253","remote_addr":"[fd00:1122:3344:101::c]:43185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":347,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.921959176Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.922086198Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"014f95be-4c23-4319-8d5f-be19b9a88577","remote_addr":"[fd00:1122:3344:101::b]:49396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.993367536Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:58.993402767Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"158447ab-5c7d-4338-9ead-9fd4c9dd7e49","remote_addr":"[fd00:1122:3344:101::b]:49396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.097629004Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.097848067Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a307ed70-abbf-409e-ad93-9a288a5fb894","remote_addr":"[fd00:1122:3344:101::b]:44192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.109971324Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3febfae6-7b77-48a3-8c99-8c6381c15bb7","remote_addr":"[fd00:1122:3344:101::b]:49396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.175552425Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33530"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.1756728Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8cf6453b-b9b5-4335-a7d1-3908e2a104e4","remote_addr":"[fd00:1122:3344:101::c]:33530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.196984715Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56544"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.197164348Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.197199798Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.197213989Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.197224239Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.197260079Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.19727945Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.19728643Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.19729849Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f626bbe0-5d75-479e-b967-9c4585db0d4e","remote_addr":"[fd00:1122:3344:101::b]:56544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":274,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.205107211Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53084"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.205191822Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.252012354Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.323458855Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"98f548bf-3ac7-4559-a853-8a925e329810","remote_addr":"[fd00:1122:3344:101::b]:53084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.323707079Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.32380619Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43019"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.323868391Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47048"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.323928492Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61435"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.324013713Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.324041824Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f717efc2-1990-4cb5-8a5b-2c560bbfeae3","remote_addr":"[fd00:1122:3344:101::a]:58375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.324058884Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"285b39a2-1bf3-4d7f-b7bd-0ed85cae56bf","remote_addr":"[fd00:1122:3344:101::b]:53084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":298,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.324333308Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.334545576Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8bcb3c39-acf6-4a49-ad4f-2f7d2d378cb4","remote_addr":"[fd00:1122:3344:101::a]:47048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":10238,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.334712328Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.334760789Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.33480843Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.33482542Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.33484129Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.334874061Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.334901601Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.335024563Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c7f105e1-b0ed-462f-aff7-136ca96c92da","remote_addr":"[fd00:1122:3344:101::c]:61435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":10573,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.366273475Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.366334716Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"389ff994-9acf-4ba8-8101-f74ac17b187d","remote_addr":"[fd00:1122:3344:101::a]:58375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.366677111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.37440845Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a912444f-8653-4289-bbed-67e7c4179f7b","remote_addr":"[fd00:1122:3344:101::a]:58375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":427,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.417596876Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37489"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421647298Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a4a08bfb-6bf1-4a14-89fe-5313e50fa2ac, kind: Service { id: e40b6155-d379-4874-9119-51a498c972c1 }, name: Name(\"external-dns-e40b6155-d379-4874-9119-51a498c972c1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 75])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cf5c8a11-e00b-4517-9b8f-fea243806b27, kind: Service { id: b130f689-a354-4ce5-be2b-20413d917375 }, name: Name(\"external-dns-b130f689-a354-4ce5-be2b-20413d917375\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 354cb18b-5e4d-4c58-956f-a6844b7e5af5, kind: Service { id: b33ec370-f410-4564-a49d-41d565d13ca6 }, name: Name(\"nexus-b33ec370-f410-4564-a49d-41d565d13ca6\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 59])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 3d83ad9c-66ab-4653-8988-8a69d587f1ac, kind: Service { id: 51c1446e-49e7-4598-8832-f33ccdf3c076 }, name: Name(\"nexus-51c1446e-49e7-4598-8832-f33ccdf3c076\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 3])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f6807eef-32bb-4d43-8315-2971f57c3725, kind: Service { id: 01130983-13be-453f-8cd5-3f2c8481c6c8 }, name: Name(\"nexus-01130983-13be-453f-8cd5-3f2c8481c6c8\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.42176045Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.42178009Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421817101Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421860792Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421878162Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421920193Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.421952903Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"81196b38-2f0a-4d12-bb92-02650e528f7d","remote_addr":"[fd00:1122:3344:101::a]:37489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":640,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.429813414Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.429839065Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"553d181a-c36b-45ef-b98c-f40f38c8e212","remote_addr":"[fd00:1122:3344:101::c]:43019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105683,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.434141721Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3f73276c-2e23-4331-a83a-f33a99b786df","remote_addr":"[fd00:1122:3344:101::c]:43019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":759,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.467930872Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.524969001Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3205a7d2-312f-4d0c-9e48-8f2bd4d1a7db","remote_addr":"[fd00:1122:3344:101::a]:45343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T18:59:59.525324936Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fb456cbe-0ce2-4e7e-ad71-8ed01fa5103e","remote_addr":"[fd00:1122:3344:101::a]:45343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.027885585Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.028106049Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7294e1f0-075b-467b-95e9-b01327c5131d","remote_addr":"[fd00:1122:3344:101::a]:37249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.02946019Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11383760) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.031518481Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"PUT","req_id":"78f0c0b5-8083-4daa-821a-8acf5cd28c88","remote_addr":"[fd00:1122:3344:101::a]:37249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3221,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.110087312Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60810"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.115568987Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 68627db6-53a8-4390-a3e6-e94f65d72ce7, project_id: 76e3412a-829f-45e7-a8cf-d891529a31cf }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-24T19:00:05.107400881Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11383760))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-977e3142880\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: f2eeb738-83cc-4507-82e5-53ed6ce8999e, read_only_parent: Some(Volume { block_size: 512, id: b16d1e38-7331-4ae5-99b3-bc4f4542879a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5d48c4fd-697b-4def-97ad-dc281263b130, key: Some(\"ByIfZ9kKx3AQkkhZsT5dYNbAuqAjO25gVeqOMku2Qo0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f2eeb738-83cc-4507-82e5-53ed6ce8999e, key: Some(\"rQzgBj5k98dP9LNEqdKZ4226h47oZVJgdROnG+Myaq8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-977e3142880\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.115636528Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.115646758Z","hostname":"buskin","pid":1572,"propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:1232","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11383760))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ccf2364a-245f-4421-aebc-9a7b72c5bb68, kind: Instance { id: 4adb4920-442e-4d65-bb79-cc0009bb1b22 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11383760), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-977e3142880\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: f2eeb738-83cc-4507-82e5-53ed6ce8999e, read_only_parent: Some(Volume { block_size: 512, id: b16d1e38-7331-4ae5-99b3-bc4f4542879a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5d48c4fd-697b-4def-97ad-dc281263b130, key: Some(\"ByIfZ9kKx3AQkkhZsT5dYNbAuqAjO25gVeqOMku2Qo0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f2eeb738-83cc-4507-82e5-53ed6ce8999e, key: Some(\"rQzgBj5k98dP9LNEqdKZ4226h47oZVJgdROnG+Myaq8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-977e3142880\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-24T19:00:05.107400881Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.11575328Z","hostname":"buskin","pid":1572,"uri":"/vmms/c85c899f-ce35-4683-8b9b-bf19ff626756","method":"PUT","req_id":"4cc779a4-c292-4a75-bb75-9767f30b1a86","remote_addr":"[fd00:1122:3344:101::a]:60810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5613,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.217558969Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.218304921Z","hostname":"buskin","pid":1572,"uri":"/vmms/c85c899f-ce35-4683-8b9b-bf19ff626756/state","method":"PUT","req_id":"8b055ea9-b3d9-463b-9e38-1c9de6e29620","remote_addr":"[fd00:1122:3344:101::a]:33483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":630,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.221257396Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 11383760 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.221297157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.264757146Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.297007424Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.390996872Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55779"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.391172855Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c9452a2d-2726-40fd-95b7-bf9c261d1565","remote_addr":"[fd00:1122:3344:101::a]:55779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:05.407449636Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {ccf2364a-245f-4421-aebc-9a7b72c5bb68: {10.151.1.175: {709622ef-4307-45dd-9bd5-a14dab0b93f3}, 10.151.1.176: {709622ef-4307-45dd-9bd5-a14dab0b93f3}}} }"} {"msg":"Profile for oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:06.527748565Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:06.527964849Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 57.752542ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:06.696505327Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 123.527051ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:06.779208382Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 297.902367ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:06.945628327Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:06.963869458Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52042"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:06.964101312Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bf591705-f4f4-4009-ab99-fdcbed57d7f1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52042","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 502.871418ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:07.260668243Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 998.396961ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:07.780482405Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 506.153606ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:08.791250736Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 771.736469ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:09.309935031Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 1.035621665s","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:10.094030997Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 616.172889ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:11.14197296Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:11.792447506Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:1710"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 48.301034ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:11.80242145Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 148.727605ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:11.860377754Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 195.814251ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:12.018150026Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\") failed: Property not found. retry in 271.994204ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:12.22345161Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.507122973Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:1719"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.546632732Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:1743"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.689815119Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:893"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.690742304Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.690825275Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"df598dc6-7d3d-4fa5-b5eb-5760e42be959","remote_addr":"[fd00:1122:3344:101::a]:55779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7283403,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-10-24T19:00:12.691494605Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.691532526Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:743"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.691550466Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:765","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-24T19:00:12.691494605Z }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.691572566Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:663","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-24T19:00:12.691494605Z }, migration_in: None, migration_out: None }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-10-24T19:00:12.735097767Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.735131098Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:743"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.735158148Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:765","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-24T19:00:12.735097767Z }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.735166078Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:663","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-24T19:00:12.735097767Z }, migration_in: None, migration_out: None }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:12.771134673Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cba4ce92-0ac9-46fb-8ef8-5fd9fc4111bf","remote_addr":"[fd00:1122:3344:101::a]:55779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":436,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:13.99188268Z","hostname":"buskin","pid":1572,"uri":"/8ed5384f-0099-459e-a18a-fdd93abd3452","method":"GET","req_id":"cfc8a976-6979-435e-9481-96ceab24622d","remote_addr":"[fd00:1122:3344:101::d]:51556","local_addr":"[fd00:1122:3344:101::1]:62798","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7724,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.828315209Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.828914739Z","hostname":"buskin","pid":1572,"uri":"/vmms/c85c899f-ce35-4683-8b9b-bf19ff626756/state","method":"PUT","req_id":"cb423e77-958c-4e35-b13e-00386362496e","remote_addr":"[fd00:1122:3344:101::a]:65034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":528,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-10-24T19:00:22.829010300Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.82903271Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:743"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.829049111Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:765","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-24T19:00:22.829010300Z }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.829063191Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:663","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-24T19:00:22.829010300Z }, migration_in: None, migration_out: None }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-10-24T19:00:22.866232498Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.866253207Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:743"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.866280902Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:765","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-10-24T19:00:22.866232498Z }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.866288158Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:779","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.866295583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:22.867063597Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"7935223463878872616"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:22.867094377Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"4137231077449573982"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.870784374Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:22.870824205Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/1efc7c65-a878-483c-82d1-872a03292008/debug/bundle/zone\", \"/pool/int/982c7515-f8a4-4d54-b007-05451b94e502/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/a5a77acc-8f53-4c36-a482-41fc0a50e4bd/crypt/debug\", \"/pool/ext/915c0c40-3159-4259-9f5f-ef7ec918e13c/crypt/debug\", \"/pool/ext/7c590c44-e234-44db-965e-9f6e7745f594/crypt/debug\", \"/pool/ext/bb91d72d-293b-493d-a375-9a8656905e43/crypt/debug\", \"/pool/ext/fa33182c-5386-4788-bb69-fed5eea70499/crypt/debug\", \"/pool/ext/62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crypt/debug\", \"/pool/ext/ebb69d76-d0be-447a-903a-9633ca2c2c1a/crypt/debug\", \"/pool/ext/2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crypt/debug\", \"/pool/ext/6063ad03-661e-4d00-b0bf-ebbe587f7c8b/crypt/debug\"] }","zone_name":"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:23.50625905Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756\", bundle_id: 139ddb19-4742-47a2-82a2-8857f60095a8 }, time_created: 2024-10-24T19:00:22.870971287Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_c85c899f-ce35-4683-8b9b-bf19ff626756","v":0,"name":"SledAgent","level":40,"time":"2024-10-24T19:00:23.506355641Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:993"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.264175958Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.283014928Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(11383760), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.283055489Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(11383760), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.283424854Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:663","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-24T19:00:25.283394424Z }, migration_in: None, migration_out: None }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.326283075Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:663","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-24T19:00:25.283394424Z }, migration_in: None, migration_out: None }","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.332733364Z","hostname":"buskin","pid":1572,"instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","propolis_id":"c85c899f-ce35-4683-8b9b-bf19ff626756","instance_id":"4adb4920-442e-4d65-bb79-cc0009bb1b22","component":"InstanceManager","file":"sled-agent/src/instance.rs:930"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.834716472Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41326"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.834873644Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"286cde12-81f2-4e0e-9894-2acc017347b5","remote_addr":"[fd00:1122:3344:101::b]:41326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.852237652Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {ccf2364a-245f-4421-aebc-9a7b72c5bb68: {10.151.1.176: {709622ef-4307-45dd-9bd5-a14dab0b93f3}, 10.151.1.175: {709622ef-4307-45dd-9bd5-a14dab0b93f3}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.852269432Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b3b99869-7c83-4075-b131-6d6a55ceb2b6","remote_addr":"[fd00:1122:3344:101::b]:41326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.880575399Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.880746531Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b73b2313-2981-4adf-9c97-f0a4ed75b3e5","remote_addr":"[fd00:1122:3344:101::b]:45017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:25.884646741Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b0d00741-d095-4f94-9340-47665cb2127b","remote_addr":"[fd00:1122:3344:101::b]:41326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":225,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.112770512Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40668"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.113012853Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"635af7b5-8d75-4886-b5a5-e39b3f227c58","remote_addr":"[fd00:1122:3344:101::a]:40668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.152524135Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.152635927Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"33d0e225-bf9f-4e60-b980-53848bc289b2","remote_addr":"[fd00:1122:3344:101::a]:63158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.168056454Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.168149296Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.168157446Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"08a44bad-eb0d-492f-b1ba-652ab3b46bff","remote_addr":"[fd00:1122:3344:101::a]:63158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.175347287Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"598d6371-4582-4c91-9e6a-27edbce02c2a","remote_addr":"[fd00:1122:3344:101::a]:63158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":363,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.789415993Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.789512034Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"252c55a5-65d5-4554-8ce8-75685a451012","remote_addr":"[fd00:1122:3344:101::a]:43624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.808259663Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.808279783Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9e52e6d8-4d04-4d12-b0d7-09b61b8ea6ed","remote_addr":"[fd00:1122:3344:101::a]:43624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:27.815120149Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"63553949-6dd8-4dea-9445-5df147959af8","remote_addr":"[fd00:1122:3344:101::a]:43624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":225,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.78050925Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64757"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.780692283Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"70a75da7-e1f2-4a39-b6cb-750f7bf0c3ad","remote_addr":"[fd00:1122:3344:101::b]:64757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.829872891Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.829903891Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"03bdb65a-6088-4df4-88d4-80348e1ec470","remote_addr":"[fd00:1122:3344:101::b]:64757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.834786167Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.834883848Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"746f538d-f95c-4850-9ab5-5488267edfc5","remote_addr":"[fd00:1122:3344:101::c]:57143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.844990174Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e6cb743f-200d-411b-a74c-c01cdf3cfd80","remote_addr":"[fd00:1122:3344:101::b]:64757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":287,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.855514786Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.855537737Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a5f62d23-de86-48b1-8184-c8940ab3437b","remote_addr":"[fd00:1122:3344:101::c]:57143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.865138485Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"73b402d8-6fed-4018-abc3-e85444fab741","remote_addr":"[fd00:1122:3344:101::c]:57143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.900650202Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51304"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.900762394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:28.94721674Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.011914187Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cc67e563-ef5e-43d3-9154-ed85fbdbb6c6","remote_addr":"[fd00:1122:3344:101::b]:51304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012031619Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47981"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.01206789Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50828"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012121251Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012346314Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"04b5d1e8-18be-4b69-a5e9-3fd44906e59e","remote_addr":"[fd00:1122:3344:101::a]:52671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012493376Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7ef07542-d2bc-4689-a936-a099cd8eaa5b","remote_addr":"[fd00:1122:3344:101::c]:50828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012510497Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"edd06c56-bd4d-4ace-b288-9eebfa5ef1e8","remote_addr":"[fd00:1122:3344:101::b]:51304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":262,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.012621328Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ce5ccde6-c57f-476a-a5e1-b0ca4fedba41","remote_addr":"[fd00:1122:3344:101::b]:47981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":285,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.018429038Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56084"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.0185858Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.065133288Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.137388851Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"12ded347-6586-40ec-9252-a65d239aee8e","remote_addr":"[fd00:1122:3344:101::c]:56084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118869,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.137919659Z","hostname":"buskin","pid":1572,"sled_id":"8ed5384f-0099-459e-a18a-fdd93abd3452","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.13794841Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"370ef972-c6b9-40b2-ac7a-20d65fd77ea2","remote_addr":"[fd00:1122:3344:101::a]:52671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.138023081Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1a8052ea-b6e6-4e49-abc4-f77d8f922f2b","remote_addr":"[fd00:1122:3344:101::c]:56084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":242,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.14898803Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.149319085Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"14037ecb-4395-4e69-8eaf-a898184e6dad","remote_addr":"[fd00:1122:3344:101::a]:38235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":221,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.153575561Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c175e27f-4597-42eb-b6b3-40818c2e181f","remote_addr":"[fd00:1122:3344:101::a]:52671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.241922653Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38701"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.242059545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.281636455Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.339889223Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2e09b8a3-82e2-44be-9280-b69ac2a4efa4","remote_addr":"[fd00:1122:3344:101::a]:38701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97878,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-24T19:00:29.340174897Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e20de9ea-a19c-4ec9-a2e5-c7ed8b015397","remote_addr":"[fd00:1122:3344:101::a]:38701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"200"}